@ryanstark24/sfgraph-core 1.4.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/analyze/apex-platform.d.ts +23 -0
- package/dist/analyze/apex-platform.d.ts.map +1 -0
- package/dist/analyze/apex-platform.js +112 -0
- package/dist/analyze/apex-platform.js.map +1 -0
- package/dist/analyze/audit-graph.d.ts +34 -10
- package/dist/analyze/audit-graph.d.ts.map +1 -1
- package/dist/analyze/audit-graph.js +360 -39
- package/dist/analyze/audit-graph.js.map +1 -1
- package/dist/analyze/component-scope.d.ts +82 -0
- package/dist/analyze/component-scope.d.ts.map +1 -0
- package/dist/analyze/component-scope.js +207 -0
- package/dist/analyze/component-scope.js.map +1 -0
- package/dist/analyze/containment.d.ts +91 -0
- package/dist/analyze/containment.d.ts.map +1 -0
- package/dist/analyze/containment.js +175 -0
- package/dist/analyze/containment.js.map +1 -0
- package/dist/analyze/cross-org.d.ts +35 -0
- package/dist/analyze/cross-org.d.ts.map +1 -1
- package/dist/analyze/cross-org.js +102 -3
- package/dist/analyze/cross-org.js.map +1 -1
- package/dist/analyze/dead-code.d.ts +99 -3
- package/dist/analyze/dead-code.d.ts.map +1 -1
- package/dist/analyze/dead-code.js +282 -10
- package/dist/analyze/dead-code.js.map +1 -1
- package/dist/analyze/dependencies.d.ts +20 -1
- package/dist/analyze/dependencies.d.ts.map +1 -1
- package/dist/analyze/dependencies.js +96 -12
- package/dist/analyze/dependencies.js.map +1 -1
- package/dist/analyze/dependents.d.ts +111 -1
- package/dist/analyze/dependents.d.ts.map +1 -1
- package/dist/analyze/dependents.js +282 -20
- package/dist/analyze/dependents.js.map +1 -1
- package/dist/analyze/deployment-manifest/generate.d.ts +24 -0
- package/dist/analyze/deployment-manifest/generate.d.ts.map +1 -1
- package/dist/analyze/deployment-manifest/generate.js +40 -12
- package/dist/analyze/deployment-manifest/generate.js.map +1 -1
- package/dist/analyze/deployment-manifest/member-name-formatters.d.ts.map +1 -1
- package/dist/analyze/deployment-manifest/member-name-formatters.js +1 -0
- package/dist/analyze/deployment-manifest/member-name-formatters.js.map +1 -1
- package/dist/analyze/deployment-manifest/member-types.d.ts.map +1 -1
- package/dist/analyze/deployment-manifest/member-types.js +2 -0
- package/dist/analyze/deployment-manifest/member-types.js.map +1 -1
- package/dist/analyze/diff-seeds.d.ts +33 -0
- package/dist/analyze/diff-seeds.d.ts.map +1 -0
- package/dist/analyze/diff-seeds.js +299 -0
- package/dist/analyze/diff-seeds.js.map +1 -0
- package/dist/analyze/edge-filters.d.ts +25 -0
- package/dist/analyze/edge-filters.d.ts.map +1 -1
- package/dist/analyze/edge-filters.js +78 -2
- package/dist/analyze/edge-filters.js.map +1 -1
- package/dist/analyze/field-scope.d.ts +25 -0
- package/dist/analyze/field-scope.d.ts.map +1 -0
- package/dist/analyze/field-scope.js +94 -0
- package/dist/analyze/field-scope.js.map +1 -0
- package/dist/analyze/find-nodes.d.ts +48 -12
- package/dist/analyze/find-nodes.d.ts.map +1 -1
- package/dist/analyze/find-nodes.js +323 -12
- package/dist/analyze/find-nodes.js.map +1 -1
- package/dist/analyze/findings.d.ts.map +1 -1
- package/dist/analyze/findings.js +14 -0
- package/dist/analyze/findings.js.map +1 -1
- package/dist/analyze/git-diff.d.ts +27 -1
- package/dist/analyze/git-diff.d.ts.map +1 -1
- package/dist/analyze/git-diff.js +198 -26
- package/dist/analyze/git-diff.js.map +1 -1
- package/dist/analyze/governor.d.ts +25 -4
- package/dist/analyze/governor.d.ts.map +1 -1
- package/dist/analyze/governor.js +82 -18
- package/dist/analyze/governor.js.map +1 -1
- package/dist/analyze/index.d.ts +6 -0
- package/dist/analyze/index.d.ts.map +1 -1
- package/dist/analyze/index.js +6 -0
- package/dist/analyze/index.js.map +1 -1
- package/dist/analyze/path-to-qname.d.ts +9 -0
- package/dist/analyze/path-to-qname.d.ts.map +1 -1
- package/dist/analyze/path-to-qname.js +58 -15
- package/dist/analyze/path-to-qname.js.map +1 -1
- package/dist/analyze/populate.d.ts +40 -0
- package/dist/analyze/populate.d.ts.map +1 -1
- package/dist/analyze/populate.js +148 -48
- package/dist/analyze/populate.js.map +1 -1
- package/dist/analyze/pseudo-endpoints.d.ts +67 -0
- package/dist/analyze/pseudo-endpoints.d.ts.map +1 -0
- package/dist/analyze/pseudo-endpoints.js +78 -0
- package/dist/analyze/pseudo-endpoints.js.map +1 -0
- package/dist/analyze/security.d.ts +2 -0
- package/dist/analyze/security.d.ts.map +1 -1
- package/dist/analyze/security.js +44 -15
- package/dist/analyze/security.js.map +1 -1
- package/dist/analyze/test-coverage.d.ts +48 -0
- package/dist/analyze/test-coverage.d.ts.map +1 -1
- package/dist/analyze/test-coverage.js +72 -1
- package/dist/analyze/test-coverage.js.map +1 -1
- package/dist/analyze/wip-changed-files.d.ts +88 -0
- package/dist/analyze/wip-changed-files.d.ts.map +1 -0
- package/dist/analyze/wip-changed-files.js +422 -0
- package/dist/analyze/wip-changed-files.js.map +1 -0
- package/dist/analyze/wip-impact.d.ts +59 -0
- package/dist/analyze/wip-impact.d.ts.map +1 -1
- package/dist/analyze/wip-impact.js +199 -126
- package/dist/analyze/wip-impact.js.map +1 -1
- package/dist/domain/apex-platform.d.ts +10 -0
- package/dist/domain/apex-platform.d.ts.map +1 -0
- package/dist/domain/apex-platform.js +231 -0
- package/dist/domain/apex-platform.js.map +1 -0
- package/dist/domain/index.d.ts +2 -0
- package/dist/domain/index.d.ts.map +1 -1
- package/dist/domain/index.js +2 -0
- package/dist/domain/index.js.map +1 -1
- package/dist/domain/rel-types.d.ts +7 -0
- package/dist/domain/rel-types.d.ts.map +1 -1
- package/dist/domain/rel-types.js +7 -0
- package/dist/domain/rel-types.js.map +1 -1
- package/dist/domain/standard-sobjects.d.ts +20 -0
- package/dist/domain/standard-sobjects.d.ts.map +1 -0
- package/dist/domain/standard-sobjects.js +183 -0
- package/dist/domain/standard-sobjects.js.map +1 -0
- package/dist/embedding/cache.d.ts +41 -0
- package/dist/embedding/cache.d.ts.map +1 -0
- package/dist/embedding/cache.js +69 -0
- package/dist/embedding/cache.js.map +1 -0
- package/dist/embedding/embed.d.ts +59 -0
- package/dist/embedding/embed.d.ts.map +1 -1
- package/dist/embedding/embed.js +193 -28
- package/dist/embedding/embed.js.map +1 -1
- package/dist/embedding/index.d.ts +7 -0
- package/dist/embedding/index.d.ts.map +1 -1
- package/dist/embedding/index.js +4 -0
- package/dist/embedding/index.js.map +1 -1
- package/dist/embedding/pool.d.ts +66 -6
- package/dist/embedding/pool.d.ts.map +1 -1
- package/dist/embedding/pool.js +156 -6
- package/dist/embedding/pool.js.map +1 -1
- package/dist/embedding/queue.d.ts +166 -9
- package/dist/embedding/queue.d.ts.map +1 -1
- package/dist/embedding/queue.js +397 -39
- package/dist/embedding/queue.js.map +1 -1
- package/dist/embedding/tokenize.d.ts +18 -0
- package/dist/embedding/tokenize.d.ts.map +1 -0
- package/dist/embedding/tokenize.js +43 -0
- package/dist/embedding/tokenize.js.map +1 -0
- package/dist/embedding/worker.d.ts +0 -5
- package/dist/embedding/worker.d.ts.map +1 -1
- package/dist/embedding/worker.js +27 -4
- package/dist/embedding/worker.js.map +1 -1
- package/dist/extractors/filesystem/index.d.ts +21 -1
- package/dist/extractors/filesystem/index.d.ts.map +1 -1
- package/dist/extractors/filesystem/index.js +195 -38
- package/dist/extractors/filesystem/index.js.map +1 -1
- package/dist/extractors/filesystem/vlocity-datapacks.d.ts +27 -0
- package/dist/extractors/filesystem/vlocity-datapacks.d.ts.map +1 -0
- package/dist/extractors/filesystem/vlocity-datapacks.js +201 -0
- package/dist/extractors/filesystem/vlocity-datapacks.js.map +1 -0
- package/dist/extractors/interfaces/metadata-source.d.ts +29 -0
- package/dist/extractors/interfaces/metadata-source.d.ts.map +1 -1
- package/dist/extractors/live-org/bulk-retrieve.d.ts +37 -1
- package/dist/extractors/live-org/bulk-retrieve.d.ts.map +1 -1
- package/dist/extractors/live-org/bulk-retrieve.js +233 -50
- package/dist/extractors/live-org/bulk-retrieve.js.map +1 -1
- package/dist/extractors/live-org/capabilities.d.ts +15 -0
- package/dist/extractors/live-org/capabilities.d.ts.map +1 -1
- package/dist/extractors/live-org/capabilities.js +54 -9
- package/dist/extractors/live-org/capabilities.js.map +1 -1
- package/dist/extractors/live-org/extractors/apex.d.ts +12 -2
- package/dist/extractors/live-org/extractors/apex.d.ts.map +1 -1
- package/dist/extractors/live-org/extractors/apex.js +151 -80
- package/dist/extractors/live-org/extractors/apex.js.map +1 -1
- package/dist/extractors/live-org/extractors/custom-settings.d.ts +2 -1
- package/dist/extractors/live-org/extractors/custom-settings.d.ts.map +1 -1
- package/dist/extractors/live-org/extractors/custom-settings.js +19 -9
- package/dist/extractors/live-org/extractors/custom-settings.js.map +1 -1
- package/dist/extractors/live-org/extractors/flow.d.ts +2 -1
- package/dist/extractors/live-org/extractors/flow.d.ts.map +1 -1
- package/dist/extractors/live-org/extractors/flow.js +12 -3
- package/dist/extractors/live-org/extractors/flow.js.map +1 -1
- package/dist/extractors/live-org/extractors/generic-metadata.d.ts +23 -3
- package/dist/extractors/live-org/extractors/generic-metadata.d.ts.map +1 -1
- package/dist/extractors/live-org/extractors/generic-metadata.js +110 -25
- package/dist/extractors/live-org/extractors/generic-metadata.js.map +1 -1
- package/dist/extractors/live-org/extractors/integration.d.ts +2 -1
- package/dist/extractors/live-org/extractors/integration.d.ts.map +1 -1
- package/dist/extractors/live-org/extractors/integration.js +14 -5
- package/dist/extractors/live-org/extractors/integration.js.map +1 -1
- package/dist/extractors/live-org/extractors/lwc.d.ts +2 -1
- package/dist/extractors/live-org/extractors/lwc.d.ts.map +1 -1
- package/dist/extractors/live-org/extractors/lwc.js +14 -6
- package/dist/extractors/live-org/extractors/lwc.js.map +1 -1
- package/dist/extractors/live-org/extractors/object.d.ts +2 -11
- package/dist/extractors/live-org/extractors/object.d.ts.map +1 -1
- package/dist/extractors/live-org/extractors/object.js +127 -226
- package/dist/extractors/live-org/extractors/object.js.map +1 -1
- package/dist/extractors/live-org/extractors/omnistudio.d.ts +3 -0
- package/dist/extractors/live-org/extractors/omnistudio.d.ts.map +1 -1
- package/dist/extractors/live-org/extractors/omnistudio.js +57 -22
- package/dist/extractors/live-org/extractors/omnistudio.js.map +1 -1
- package/dist/extractors/live-org/extractors/security.d.ts +12 -1
- package/dist/extractors/live-org/extractors/security.d.ts.map +1 -1
- package/dist/extractors/live-org/extractors/security.js +29 -7
- package/dist/extractors/live-org/extractors/security.js.map +1 -1
- package/dist/extractors/live-org/rate-limit.d.ts +7 -1
- package/dist/extractors/live-org/rate-limit.d.ts.map +1 -1
- package/dist/extractors/live-org/rate-limit.js +53 -10
- package/dist/extractors/live-org/rate-limit.js.map +1 -1
- package/dist/extractors/live-org/retry.d.ts +136 -0
- package/dist/extractors/live-org/retry.d.ts.map +1 -0
- package/dist/extractors/live-org/retry.js +416 -0
- package/dist/extractors/live-org/retry.js.map +1 -0
- package/dist/extractors/live-org/vlocity/runner.d.ts +2 -1
- package/dist/extractors/live-org/vlocity/runner.d.ts.map +1 -1
- package/dist/extractors/live-org/vlocity/runner.js +95 -67
- package/dist/extractors/live-org/vlocity/runner.js.map +1 -1
- package/dist/extractors/vlocity-shared/row.d.ts +19 -0
- package/dist/extractors/vlocity-shared/row.d.ts.map +1 -0
- package/dist/extractors/vlocity-shared/row.js +69 -0
- package/dist/extractors/vlocity-shared/row.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/ingest/delta.d.ts +70 -0
- package/dist/ingest/delta.d.ts.map +1 -0
- package/dist/ingest/delta.js +534 -0
- package/dist/ingest/delta.js.map +1 -0
- package/dist/ingest/detect-deletions-guard.d.ts +34 -0
- package/dist/ingest/detect-deletions-guard.d.ts.map +1 -1
- package/dist/ingest/detect-deletions-guard.js +59 -0
- package/dist/ingest/detect-deletions-guard.js.map +1 -1
- package/dist/ingest/index.d.ts +8 -1
- package/dist/ingest/index.d.ts.map +1 -1
- package/dist/ingest/index.js +4 -0
- package/dist/ingest/index.js.map +1 -1
- package/dist/ingest/live-ingest.d.ts +171 -7
- package/dist/ingest/live-ingest.d.ts.map +1 -1
- package/dist/ingest/live-ingest.js +1014 -249
- package/dist/ingest/live-ingest.js.map +1 -1
- package/dist/ingest/lock.d.ts +47 -0
- package/dist/ingest/lock.d.ts.map +1 -0
- package/dist/ingest/lock.js +222 -0
- package/dist/ingest/lock.js.map +1 -0
- package/dist/ingest/member-rows.d.ts +88 -0
- package/dist/ingest/member-rows.d.ts.map +1 -0
- package/dist/ingest/member-rows.js +152 -0
- package/dist/ingest/member-rows.js.map +1 -0
- package/dist/ingest/offline-org.d.ts +8 -0
- package/dist/ingest/offline-org.d.ts.map +1 -0
- package/dist/ingest/offline-org.js +41 -0
- package/dist/ingest/offline-org.js.map +1 -0
- package/dist/ingest/parse-member.d.ts +53 -0
- package/dist/ingest/parse-member.d.ts.map +1 -0
- package/dist/ingest/parse-member.js +276 -0
- package/dist/ingest/parse-member.js.map +1 -0
- package/dist/ingest/parse-pool.d.ts +139 -0
- package/dist/ingest/parse-pool.d.ts.map +1 -0
- package/dist/ingest/parse-pool.js +593 -0
- package/dist/ingest/parse-pool.js.map +1 -0
- package/dist/ingest/parse-worker.d.ts +3 -0
- package/dist/ingest/parse-worker.d.ts.map +1 -0
- package/dist/ingest/parse-worker.js +100 -0
- package/dist/ingest/parse-worker.js.map +1 -0
- package/dist/ingest/profile.d.ts +27 -0
- package/dist/ingest/profile.d.ts.map +1 -0
- package/dist/ingest/profile.js +48 -0
- package/dist/ingest/profile.js.map +1 -0
- package/dist/ingest/resolver-gates.d.ts +12 -0
- package/dist/ingest/resolver-gates.d.ts.map +1 -0
- package/dist/ingest/resolver-gates.js +53 -0
- package/dist/ingest/resolver-gates.js.map +1 -0
- package/dist/parsers/apex/arity-resolver.d.ts +16 -3
- package/dist/parsers/apex/arity-resolver.d.ts.map +1 -1
- package/dist/parsers/apex/arity-resolver.js +31 -2
- package/dist/parsers/apex/arity-resolver.js.map +1 -1
- package/dist/parsers/apex/ast-extractor.d.ts +4 -0
- package/dist/parsers/apex/ast-extractor.d.ts.map +1 -1
- package/dist/parsers/apex/ast-extractor.js +96 -29
- package/dist/parsers/apex/ast-extractor.js.map +1 -1
- package/dist/parsers/apex/class-refs.d.ts +78 -0
- package/dist/parsers/apex/class-refs.d.ts.map +1 -0
- package/dist/parsers/apex/class-refs.js +465 -0
- package/dist/parsers/apex/class-refs.js.map +1 -0
- package/dist/parsers/apex/class.d.ts +158 -0
- package/dist/parsers/apex/class.d.ts.map +1 -1
- package/dist/parsers/apex/class.js +1025 -183
- package/dist/parsers/apex/class.js.map +1 -1
- package/dist/parsers/apex/dynamic-soql.d.ts +52 -0
- package/dist/parsers/apex/dynamic-soql.d.ts.map +1 -0
- package/dist/parsers/apex/dynamic-soql.js +427 -0
- package/dist/parsers/apex/dynamic-soql.js.map +1 -0
- package/dist/parsers/apex/field-access.d.ts +65 -0
- package/dist/parsers/apex/field-access.d.ts.map +1 -0
- package/dist/parsers/apex/field-access.js +438 -0
- package/dist/parsers/apex/field-access.js.map +1 -0
- package/dist/parsers/apex/interface-ref-resolver.d.ts +32 -0
- package/dist/parsers/apex/interface-ref-resolver.d.ts.map +1 -0
- package/dist/parsers/apex/interface-ref-resolver.js +64 -0
- package/dist/parsers/apex/interface-ref-resolver.js.map +1 -0
- package/dist/parsers/apex/name-refs.d.ts +83 -0
- package/dist/parsers/apex/name-refs.d.ts.map +1 -0
- package/dist/parsers/apex/name-refs.js +465 -0
- package/dist/parsers/apex/name-refs.js.map +1 -0
- package/dist/parsers/apex/page-refs.d.ts +11 -0
- package/dist/parsers/apex/page-refs.d.ts.map +1 -0
- package/dist/parsers/apex/page-refs.js +34 -0
- package/dist/parsers/apex/page-refs.js.map +1 -0
- package/dist/parsers/apex/ref-resolver.d.ts +58 -0
- package/dist/parsers/apex/ref-resolver.d.ts.map +1 -0
- package/dist/parsers/apex/ref-resolver.js +348 -0
- package/dist/parsers/apex/ref-resolver.js.map +1 -0
- package/dist/parsers/apex/soql-fields.d.ts +75 -0
- package/dist/parsers/apex/soql-fields.d.ts.map +1 -0
- package/dist/parsers/apex/soql-fields.js +338 -0
- package/dist/parsers/apex/soql-fields.js.map +1 -0
- package/dist/parsers/apex/trigger.d.ts.map +1 -1
- package/dist/parsers/apex/trigger.js +90 -2
- package/dist/parsers/apex/trigger.js.map +1 -1
- package/dist/parsers/apex/vlocity-calls.d.ts +25 -0
- package/dist/parsers/apex/vlocity-calls.d.ts.map +1 -0
- package/dist/parsers/apex/vlocity-calls.js +225 -0
- package/dist/parsers/apex/vlocity-calls.js.map +1 -0
- package/dist/parsers/aura/bundle.d.ts +39 -0
- package/dist/parsers/aura/bundle.d.ts.map +1 -0
- package/dist/parsers/aura/bundle.js +269 -0
- package/dist/parsers/aura/bundle.js.map +1 -0
- package/dist/parsers/aura/embed-resolver.d.ts +17 -0
- package/dist/parsers/aura/embed-resolver.d.ts.map +1 -0
- package/dist/parsers/aura/embed-resolver.js +58 -0
- package/dist/parsers/aura/embed-resolver.js.map +1 -0
- package/dist/parsers/aura/index.d.ts +4 -0
- package/dist/parsers/aura/index.d.ts.map +1 -0
- package/dist/parsers/aura/index.js +6 -0
- package/dist/parsers/aura/index.js.map +1 -0
- package/dist/parsers/common.d.ts +9 -0
- package/dist/parsers/common.d.ts.map +1 -1
- package/dist/parsers/common.js +17 -0
- package/dist/parsers/common.js.map +1 -1
- package/dist/parsers/cross-flavor-resolver.js +8 -2
- package/dist/parsers/cross-flavor-resolver.js.map +1 -1
- package/dist/parsers/flow/invocable-resolver.d.ts.map +1 -1
- package/dist/parsers/flow/invocable-resolver.js +2 -0
- package/dist/parsers/flow/invocable-resolver.js.map +1 -1
- package/dist/parsers/flow/parser.d.ts +9 -0
- package/dist/parsers/flow/parser.d.ts.map +1 -1
- package/dist/parsers/flow/parser.js +327 -2
- package/dist/parsers/flow/parser.js.map +1 -1
- package/dist/parsers/flow/relationship-resolver.d.ts +34 -0
- package/dist/parsers/flow/relationship-resolver.d.ts.map +1 -0
- package/dist/parsers/flow/relationship-resolver.js +199 -0
- package/dist/parsers/flow/relationship-resolver.js.map +1 -0
- package/dist/parsers/generic/opaque-node-parser.d.ts +5 -1
- package/dist/parsers/generic/opaque-node-parser.d.ts.map +1 -1
- package/dist/parsers/generic/opaque-node-parser.js +37 -5
- package/dist/parsers/generic/opaque-node-parser.js.map +1 -1
- package/dist/parsers/generic/reflection-walker.d.ts +7 -0
- package/dist/parsers/generic/reflection-walker.d.ts.map +1 -1
- package/dist/parsers/generic/reflection-walker.js +37 -10
- package/dist/parsers/generic/reflection-walker.js.map +1 -1
- package/dist/parsers/hash-salts.d.ts +72 -0
- package/dist/parsers/hash-salts.d.ts.map +1 -0
- package/dist/parsers/hash-salts.js +91 -0
- package/dist/parsers/hash-salts.js.map +1 -0
- package/dist/parsers/index.d.ts +2 -0
- package/dist/parsers/index.d.ts.map +1 -1
- package/dist/parsers/index.js +2 -0
- package/dist/parsers/index.js.map +1 -1
- package/dist/parsers/lwc/bundle.d.ts +1 -0
- package/dist/parsers/lwc/bundle.d.ts.map +1 -1
- package/dist/parsers/lwc/bundle.js +37 -2
- package/dist/parsers/lwc/bundle.js.map +1 -1
- package/dist/parsers/lwc/field-key-resolver.d.ts +32 -0
- package/dist/parsers/lwc/field-key-resolver.d.ts.map +1 -0
- package/dist/parsers/lwc/field-key-resolver.js +92 -0
- package/dist/parsers/lwc/field-key-resolver.js.map +1 -0
- package/dist/parsers/lwc/html-visitor.d.ts +1 -1
- package/dist/parsers/lwc/html-visitor.d.ts.map +1 -1
- package/dist/parsers/lwc/html-visitor.js +78 -1
- package/dist/parsers/lwc/html-visitor.js.map +1 -1
- package/dist/parsers/lwc/js-visitor.d.ts +16 -0
- package/dist/parsers/lwc/js-visitor.d.ts.map +1 -1
- package/dist/parsers/lwc/js-visitor.js +348 -10
- package/dist/parsers/lwc/js-visitor.js.map +1 -1
- package/dist/parsers/lwc/omnidef.d.ts +10 -0
- package/dist/parsers/lwc/omnidef.d.ts.map +1 -0
- package/dist/parsers/lwc/omnidef.js +127 -0
- package/dist/parsers/lwc/omnidef.js.map +1 -0
- package/dist/parsers/lwc/xml-meta.d.ts.map +1 -1
- package/dist/parsers/lwc/xml-meta.js +16 -1
- package/dist/parsers/lwc/xml-meta.js.map +1 -1
- package/dist/parsers/object/formula-hop-resolver.d.ts +33 -0
- package/dist/parsers/object/formula-hop-resolver.d.ts.map +1 -0
- package/dist/parsers/object/formula-hop-resolver.js +60 -0
- package/dist/parsers/object/formula-hop-resolver.js.map +1 -0
- package/dist/parsers/object/formula-refs.d.ts +40 -0
- package/dist/parsers/object/formula-refs.d.ts.map +1 -0
- package/dist/parsers/object/formula-refs.js +129 -0
- package/dist/parsers/object/formula-refs.js.map +1 -0
- package/dist/parsers/object/object.d.ts +3 -0
- package/dist/parsers/object/object.d.ts.map +1 -1
- package/dist/parsers/object/object.js +53 -52
- package/dist/parsers/object/object.js.map +1 -1
- package/dist/parsers/omnistudio/data-transform.d.ts.map +1 -1
- package/dist/parsers/omnistudio/data-transform.js +2 -2
- package/dist/parsers/omnistudio/data-transform.js.map +1 -1
- package/dist/parsers/omnistudio/integration-procedure.d.ts.map +1 -1
- package/dist/parsers/omnistudio/integration-procedure.js +4 -5
- package/dist/parsers/omnistudio/integration-procedure.js.map +1 -1
- package/dist/parsers/omnistudio/overlap-detector.d.ts.map +1 -1
- package/dist/parsers/omnistudio/overlap-detector.js +27 -2
- package/dist/parsers/omnistudio/overlap-detector.js.map +1 -1
- package/dist/parsers/omnistudio/process.d.ts.map +1 -1
- package/dist/parsers/omnistudio/process.js +4 -5
- package/dist/parsers/omnistudio/process.js.map +1 -1
- package/dist/parsers/omnistudio/remote-refs.d.ts +11 -0
- package/dist/parsers/omnistudio/remote-refs.d.ts.map +1 -0
- package/dist/parsers/omnistudio/remote-refs.js +17 -0
- package/dist/parsers/omnistudio/remote-refs.js.map +1 -0
- package/dist/parsers/omnistudio/ui-card.d.ts.map +1 -1
- package/dist/parsers/omnistudio/ui-card.js +4 -5
- package/dist/parsers/omnistudio/ui-card.js.map +1 -1
- package/dist/parsers/resolver-prune.d.ts +14 -0
- package/dist/parsers/resolver-prune.d.ts.map +1 -0
- package/dist/parsers/resolver-prune.js +29 -0
- package/dist/parsers/resolver-prune.js.map +1 -0
- package/dist/parsers/rules/_engine.d.ts.map +1 -1
- package/dist/parsers/rules/_engine.js +4 -0
- package/dist/parsers/rules/_engine.js.map +1 -1
- package/dist/parsers/rules/_engine.ts +3 -0
- package/dist/parsers/rules/_loader.d.ts +8 -0
- package/dist/parsers/rules/_loader.d.ts.map +1 -1
- package/dist/parsers/rules/_loader.js +17 -0
- package/dist/parsers/rules/_loader.js.map +1 -1
- package/dist/parsers/rules/_loader.ts +19 -0
- package/dist/parsers/rules/_schema.d.ts +12 -12
- package/dist/parsers/rules/_selectors.d.ts +1 -0
- package/dist/parsers/rules/_selectors.d.ts.map +1 -1
- package/dist/parsers/rules/_selectors.js +6 -0
- package/dist/parsers/rules/_selectors.js.map +1 -1
- package/dist/parsers/rules/_selectors.ts +6 -0
- package/dist/parsers/rules/apex-component.yml +49 -4
- package/dist/parsers/rules/apex-page.yml +49 -4
- package/dist/parsers/rules/approval-process.yml +25 -5
- package/dist/parsers/rules/custom-metadata-type.yml +7 -7
- package/dist/parsers/rules/custom-tab.yml +41 -0
- package/dist/parsers/rules/lightning-page.yml +4 -2
- package/dist/parsers/rules/workflow.yml +30 -10
- package/dist/parsers/vf/refs.d.ts +39 -0
- package/dist/parsers/vf/refs.d.ts.map +1 -0
- package/dist/parsers/vf/refs.js +152 -0
- package/dist/parsers/vf/refs.js.map +1 -0
- package/dist/parsers/vlocity/apex-resolver.d.ts +21 -0
- package/dist/parsers/vlocity/apex-resolver.d.ts.map +1 -0
- package/dist/parsers/vlocity/apex-resolver.js +115 -0
- package/dist/parsers/vlocity/apex-resolver.js.map +1 -0
- package/dist/parsers/vlocity/common.d.ts +56 -1
- package/dist/parsers/vlocity/common.d.ts.map +1 -1
- package/dist/parsers/vlocity/common.js +275 -7
- package/dist/parsers/vlocity/common.js.map +1 -1
- package/dist/parsers/vlocity/data-raptor.d.ts +9 -0
- package/dist/parsers/vlocity/data-raptor.d.ts.map +1 -1
- package/dist/parsers/vlocity/data-raptor.js +133 -31
- package/dist/parsers/vlocity/data-raptor.js.map +1 -1
- package/dist/parsers/vlocity/datapack-refs.d.ts +22 -0
- package/dist/parsers/vlocity/datapack-refs.d.ts.map +1 -0
- package/dist/parsers/vlocity/datapack-refs.js +126 -0
- package/dist/parsers/vlocity/datapack-refs.js.map +1 -0
- package/dist/parsers/vlocity/dr-relationship-resolver.d.ts +26 -0
- package/dist/parsers/vlocity/dr-relationship-resolver.d.ts.map +1 -0
- package/dist/parsers/vlocity/dr-relationship-resolver.js +137 -0
- package/dist/parsers/vlocity/dr-relationship-resolver.js.map +1 -0
- package/dist/parsers/vlocity/integration-procedure.d.ts.map +1 -1
- package/dist/parsers/vlocity/integration-procedure.js +15 -48
- package/dist/parsers/vlocity/integration-procedure.js.map +1 -1
- package/dist/parsers/vlocity/name-match-resolver.d.ts +21 -0
- package/dist/parsers/vlocity/name-match-resolver.d.ts.map +1 -0
- package/dist/parsers/vlocity/name-match-resolver.js +97 -0
- package/dist/parsers/vlocity/name-match-resolver.js.map +1 -0
- package/dist/parsers/vlocity/omni-script.d.ts.map +1 -1
- package/dist/parsers/vlocity/omni-script.js +18 -32
- package/dist/parsers/vlocity/omni-script.js.map +1 -1
- package/dist/parsers/vlocity/vlocity-card.d.ts.map +1 -1
- package/dist/parsers/vlocity/vlocity-card.js +30 -36
- package/dist/parsers/vlocity/vlocity-card.js.map +1 -1
- package/dist/render/mermaid/dependency-graph.d.ts.map +1 -1
- package/dist/render/mermaid/dependency-graph.js +13 -1
- package/dist/render/mermaid/dependency-graph.js.map +1 -1
- package/dist/render/mermaid/diff.d.ts.map +1 -1
- package/dist/render/mermaid/diff.js +4 -2
- package/dist/render/mermaid/diff.js.map +1 -1
- package/dist/render/mermaid/shared.d.ts +7 -0
- package/dist/render/mermaid/shared.d.ts.map +1 -1
- package/dist/render/mermaid/shared.js +15 -6
- package/dist/render/mermaid/shared.js.map +1 -1
- package/dist/storage/index.d.ts +3 -3
- package/dist/storage/index.d.ts.map +1 -1
- package/dist/storage/index.js +2 -2
- package/dist/storage/index.js.map +1 -1
- package/dist/storage/interfaces.d.ts +155 -1
- package/dist/storage/interfaces.d.ts.map +1 -1
- package/dist/storage/sqlite/graph-store.d.ts +352 -3
- package/dist/storage/sqlite/graph-store.d.ts.map +1 -1
- package/dist/storage/sqlite/graph-store.js +1683 -74
- package/dist/storage/sqlite/graph-store.js.map +1 -1
- package/dist/storage/sqlite/migrations.d.ts +11 -0
- package/dist/storage/sqlite/migrations.d.ts.map +1 -1
- package/dist/storage/sqlite/migrations.js +226 -11
- package/dist/storage/sqlite/migrations.js.map +1 -1
- package/dist/storage/sqlite/snapshot-store.d.ts +79 -3
- package/dist/storage/sqlite/snapshot-store.d.ts.map +1 -1
- package/dist/storage/sqlite/snapshot-store.js +500 -100
- package/dist/storage/sqlite/snapshot-store.js.map +1 -1
- package/dist/storage/sqlite/snapshot-versions.d.ts +51 -0
- package/dist/storage/sqlite/snapshot-versions.d.ts.map +1 -0
- package/dist/storage/sqlite/snapshot-versions.js +209 -0
- package/dist/storage/sqlite/snapshot-versions.js.map +1 -0
- package/dist/storage/sqlite/vector-store.d.ts +12 -0
- package/dist/storage/sqlite/vector-store.d.ts.map +1 -1
- package/dist/storage/sqlite/vector-store.js +57 -14
- package/dist/storage/sqlite/vector-store.js.map +1 -1
- package/dist/util/process-owner.d.ts +35 -0
- package/dist/util/process-owner.d.ts.map +1 -0
- package/dist/util/process-owner.js +87 -0
- package/dist/util/process-owner.js.map +1 -0
- package/dist/util/worker-heap-cap.d.ts +10 -0
- package/dist/util/worker-heap-cap.d.ts.map +1 -0
- package/dist/util/worker-heap-cap.js +65 -0
- package/dist/util/worker-heap-cap.js.map +1 -0
- package/package.json +6 -7
- package/dist/analyze/__tests__/audit-graph.test.d.ts +0 -2
- package/dist/analyze/__tests__/audit-graph.test.d.ts.map +0 -1
- package/dist/analyze/__tests__/audit-graph.test.js +0 -201
- package/dist/analyze/__tests__/audit-graph.test.js.map +0 -1
- package/dist/analyze/__tests__/cross-org-two-store.test.d.ts +0 -2
- package/dist/analyze/__tests__/cross-org-two-store.test.d.ts.map +0 -1
- package/dist/analyze/__tests__/cross-org-two-store.test.js +0 -68
- package/dist/analyze/__tests__/cross-org-two-store.test.js.map +0 -1
- package/dist/analyze/__tests__/dead-code-confidence.test.d.ts +0 -2
- package/dist/analyze/__tests__/dead-code-confidence.test.d.ts.map +0 -1
- package/dist/analyze/__tests__/dead-code-confidence.test.js +0 -66
- package/dist/analyze/__tests__/dead-code-confidence.test.js.map +0 -1
- package/dist/analyze/__tests__/deployment-manifest.test.d.ts +0 -2
- package/dist/analyze/__tests__/deployment-manifest.test.d.ts.map +0 -1
- package/dist/analyze/__tests__/deployment-manifest.test.js +0 -66
- package/dist/analyze/__tests__/deployment-manifest.test.js.map +0 -1
- package/dist/analyze/__tests__/edge-filters.test.d.ts +0 -2
- package/dist/analyze/__tests__/edge-filters.test.d.ts.map +0 -1
- package/dist/analyze/__tests__/edge-filters.test.js +0 -66
- package/dist/analyze/__tests__/edge-filters.test.js.map +0 -1
- package/dist/analyze/__tests__/find-nodes.test.d.ts +0 -2
- package/dist/analyze/__tests__/find-nodes.test.d.ts.map +0 -1
- package/dist/analyze/__tests__/find-nodes.test.js +0 -129
- package/dist/analyze/__tests__/find-nodes.test.js.map +0 -1
- package/dist/analyze/__tests__/findings.test.d.ts +0 -2
- package/dist/analyze/__tests__/findings.test.d.ts.map +0 -1
- package/dist/analyze/__tests__/findings.test.js +0 -100
- package/dist/analyze/__tests__/findings.test.js.map +0 -1
- package/dist/analyze/__tests__/governor-risks-detection.test.d.ts +0 -2
- package/dist/analyze/__tests__/governor-risks-detection.test.d.ts.map +0 -1
- package/dist/analyze/__tests__/governor-risks-detection.test.js +0 -160
- package/dist/analyze/__tests__/governor-risks-detection.test.js.map +0 -1
- package/dist/analyze/__tests__/populate.test.d.ts +0 -2
- package/dist/analyze/__tests__/populate.test.d.ts.map +0 -1
- package/dist/analyze/__tests__/populate.test.js +0 -129
- package/dist/analyze/__tests__/populate.test.js.map +0 -1
- package/dist/analyze/__tests__/security-fls-filter.test.d.ts +0 -2
- package/dist/analyze/__tests__/security-fls-filter.test.d.ts.map +0 -1
- package/dist/analyze/__tests__/security-fls-filter.test.js +0 -23
- package/dist/analyze/__tests__/security-fls-filter.test.js.map +0 -1
- package/dist/analyze/__tests__/traversal-caps.test.d.ts +0 -2
- package/dist/analyze/__tests__/traversal-caps.test.d.ts.map +0 -1
- package/dist/analyze/__tests__/traversal-caps.test.js +0 -129
- package/dist/analyze/__tests__/traversal-caps.test.js.map +0 -1
- package/dist/analyze/__tests__/wip-impact.test.d.ts +0 -2
- package/dist/analyze/__tests__/wip-impact.test.d.ts.map +0 -1
- package/dist/analyze/__tests__/wip-impact.test.js +0 -134
- package/dist/analyze/__tests__/wip-impact.test.js.map +0 -1
- package/dist/embedding/__tests__/embed.test.d.ts +0 -2
- package/dist/embedding/__tests__/embed.test.d.ts.map +0 -1
- package/dist/embedding/__tests__/embed.test.js +0 -72
- package/dist/embedding/__tests__/embed.test.js.map +0 -1
- package/dist/embedding/__tests__/queue.test.d.ts +0 -2
- package/dist/embedding/__tests__/queue.test.d.ts.map +0 -1
- package/dist/embedding/__tests__/queue.test.js +0 -64
- package/dist/embedding/__tests__/queue.test.js.map +0 -1
- package/dist/extractors/__tests__/network-egress.test.d.ts +0 -2
- package/dist/extractors/__tests__/network-egress.test.d.ts.map +0 -1
- package/dist/extractors/__tests__/network-egress.test.js +0 -32
- package/dist/extractors/__tests__/network-egress.test.js.map +0 -1
- package/dist/extractors/filesystem/__tests__/filesystem.test.d.ts +0 -2
- package/dist/extractors/filesystem/__tests__/filesystem.test.d.ts.map +0 -1
- package/dist/extractors/filesystem/__tests__/filesystem.test.js +0 -96
- package/dist/extractors/filesystem/__tests__/filesystem.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/_jsforce-mock.d.ts +0 -38
- package/dist/extractors/live-org/__tests__/_jsforce-mock.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/_jsforce-mock.js +0 -96
- package/dist/extractors/live-org/__tests__/_jsforce-mock.js.map +0 -1
- package/dist/extractors/live-org/__tests__/auth.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/auth.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/auth.test.js +0 -85
- package/dist/extractors/live-org/__tests__/auth.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/bulk-retrieve.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/bulk-retrieve.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/bulk-retrieve.test.js +0 -150
- package/dist/extractors/live-org/__tests__/bulk-retrieve.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/capabilities.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/capabilities.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/capabilities.test.js +0 -54
- package/dist/extractors/live-org/__tests__/capabilities.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/custom-settings.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/custom-settings.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/custom-settings.test.js +0 -79
- package/dist/extractors/live-org/__tests__/custom-settings.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/discovery.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/discovery.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/discovery.test.js +0 -55
- package/dist/extractors/live-org/__tests__/discovery.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/dispatch.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/dispatch.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/dispatch.test.js +0 -68
- package/dist/extractors/live-org/__tests__/dispatch.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/extractor-enrichment.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/extractor-enrichment.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/extractor-enrichment.test.js +0 -374
- package/dist/extractors/live-org/__tests__/extractor-enrichment.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/liveness.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/liveness.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/liveness.test.js +0 -148
- package/dist/extractors/live-org/__tests__/liveness.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/mcd-baseline.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/mcd-baseline.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/mcd-baseline.test.js +0 -177
- package/dist/extractors/live-org/__tests__/mcd-baseline.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/metadata-read-batch.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/metadata-read-batch.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/metadata-read-batch.test.js +0 -75
- package/dist/extractors/live-org/__tests__/metadata-read-batch.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/omnistudio-retrieve.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/omnistudio-retrieve.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/omnistudio-retrieve.test.js +0 -135
- package/dist/extractors/live-org/__tests__/omnistudio-retrieve.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/pools.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/pools.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/pools.test.js +0 -78
- package/dist/extractors/live-org/__tests__/pools.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/rate-limit.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/rate-limit.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/rate-limit.test.js +0 -73
- package/dist/extractors/live-org/__tests__/rate-limit.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/read-only-proxy.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/read-only-proxy.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/read-only-proxy.test.js +0 -376
- package/dist/extractors/live-org/__tests__/read-only-proxy.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/soql-rebatch.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/soql-rebatch.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/soql-rebatch.test.js +0 -118
- package/dist/extractors/live-org/__tests__/soql-rebatch.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/source-member.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/source-member.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/source-member.test.js +0 -84
- package/dist/extractors/live-org/__tests__/source-member.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/stop-waiting.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/stop-waiting.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/stop-waiting.test.js +0 -243
- package/dist/extractors/live-org/__tests__/stop-waiting.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/vlocity-runner.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/vlocity-runner.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/vlocity-runner.test.js +0 -115
- package/dist/extractors/live-org/__tests__/vlocity-runner.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/watchdog-budgets.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/watchdog-budgets.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/watchdog-budgets.test.js +0 -40
- package/dist/extractors/live-org/__tests__/watchdog-budgets.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/wedge-isolation.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/wedge-isolation.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/wedge-isolation.test.js +0 -270
- package/dist/extractors/live-org/__tests__/wedge-isolation.test.js.map +0 -1
- package/dist/extractors/live-org/extractors/__tests__/apex.test.d.ts +0 -2
- package/dist/extractors/live-org/extractors/__tests__/apex.test.d.ts.map +0 -1
- package/dist/extractors/live-org/extractors/__tests__/apex.test.js +0 -73
- package/dist/extractors/live-org/extractors/__tests__/apex.test.js.map +0 -1
- package/dist/extractors/live-org/extractors/__tests__/lwc.parallel.test.d.ts +0 -2
- package/dist/extractors/live-org/extractors/__tests__/lwc.parallel.test.d.ts.map +0 -1
- package/dist/extractors/live-org/extractors/__tests__/lwc.parallel.test.js +0 -183
- package/dist/extractors/live-org/extractors/__tests__/lwc.parallel.test.js.map +0 -1
- package/dist/ingest/__tests__/config-values.test.d.ts +0 -2
- package/dist/ingest/__tests__/config-values.test.d.ts.map +0 -1
- package/dist/ingest/__tests__/config-values.test.js +0 -65
- package/dist/ingest/__tests__/config-values.test.js.map +0 -1
- package/dist/ingest/__tests__/full-sync-deletion.test.d.ts +0 -2
- package/dist/ingest/__tests__/full-sync-deletion.test.d.ts.map +0 -1
- package/dist/ingest/__tests__/full-sync-deletion.test.js +0 -207
- package/dist/ingest/__tests__/full-sync-deletion.test.js.map +0 -1
- package/dist/ingest/__tests__/incremental-deletion.test.d.ts +0 -2
- package/dist/ingest/__tests__/incremental-deletion.test.d.ts.map +0 -1
- package/dist/ingest/__tests__/incremental-deletion.test.js +0 -92
- package/dist/ingest/__tests__/incremental-deletion.test.js.map +0 -1
- package/dist/ingest/__tests__/live-ingest.detect-deletions-guard.test.d.ts +0 -10
- package/dist/ingest/__tests__/live-ingest.detect-deletions-guard.test.d.ts.map +0 -1
- package/dist/ingest/__tests__/live-ingest.detect-deletions-guard.test.js +0 -251
- package/dist/ingest/__tests__/live-ingest.detect-deletions-guard.test.js.map +0 -1
- package/dist/ingest/__tests__/live-ingest.test.d.ts +0 -2
- package/dist/ingest/__tests__/live-ingest.test.d.ts.map +0 -1
- package/dist/ingest/__tests__/live-ingest.test.js +0 -155
- package/dist/ingest/__tests__/live-ingest.test.js.map +0 -1
- package/dist/ingest/__tests__/multi-org-orchestrator.test.d.ts +0 -2
- package/dist/ingest/__tests__/multi-org-orchestrator.test.d.ts.map +0 -1
- package/dist/ingest/__tests__/multi-org-orchestrator.test.js +0 -108
- package/dist/ingest/__tests__/multi-org-orchestrator.test.js.map +0 -1
- package/dist/ingest/__tests__/parser-dispatch.test.d.ts +0 -2
- package/dist/ingest/__tests__/parser-dispatch.test.d.ts.map +0 -1
- package/dist/ingest/__tests__/parser-dispatch.test.js +0 -55
- package/dist/ingest/__tests__/parser-dispatch.test.js.map +0 -1
- package/dist/ingest/__tests__/read-only-wrapping.test.d.ts +0 -2
- package/dist/ingest/__tests__/read-only-wrapping.test.d.ts.map +0 -1
- package/dist/ingest/__tests__/read-only-wrapping.test.js +0 -62
- package/dist/ingest/__tests__/read-only-wrapping.test.js.map +0 -1
- package/dist/ingest/__tests__/vlocity-dispatch-integration.test.d.ts +0 -2
- package/dist/ingest/__tests__/vlocity-dispatch-integration.test.d.ts.map +0 -1
- package/dist/ingest/__tests__/vlocity-dispatch-integration.test.js +0 -134
- package/dist/ingest/__tests__/vlocity-dispatch-integration.test.js.map +0 -1
- package/dist/parsers/__tests__/_harness.d.ts +0 -13
- package/dist/parsers/__tests__/_harness.d.ts.map +0 -1
- package/dist/parsers/__tests__/_harness.js +0 -84
- package/dist/parsers/__tests__/_harness.js.map +0 -1
- package/dist/parsers/__tests__/apex.golden.test.d.ts +0 -2
- package/dist/parsers/__tests__/apex.golden.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/apex.golden.test.js +0 -28
- package/dist/parsers/__tests__/apex.golden.test.js.map +0 -1
- package/dist/parsers/__tests__/cross-flavor.test.d.ts +0 -2
- package/dist/parsers/__tests__/cross-flavor.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/cross-flavor.test.js +0 -56
- package/dist/parsers/__tests__/cross-flavor.test.js.map +0 -1
- package/dist/parsers/__tests__/datatransform-structured.test.d.ts +0 -2
- package/dist/parsers/__tests__/datatransform-structured.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/datatransform-structured.test.js +0 -147
- package/dist/parsers/__tests__/datatransform-structured.test.js.map +0 -1
- package/dist/parsers/__tests__/edge-provenance.test.d.ts +0 -2
- package/dist/parsers/__tests__/edge-provenance.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/edge-provenance.test.js +0 -47
- package/dist/parsers/__tests__/edge-provenance.test.js.map +0 -1
- package/dist/parsers/__tests__/flow.golden.test.d.ts +0 -2
- package/dist/parsers/__tests__/flow.golden.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/flow.golden.test.js +0 -17
- package/dist/parsers/__tests__/flow.golden.test.js.map +0 -1
- package/dist/parsers/__tests__/integration.golden.test.d.ts +0 -2
- package/dist/parsers/__tests__/integration.golden.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/integration.golden.test.js +0 -31
- package/dist/parsers/__tests__/integration.golden.test.js.map +0 -1
- package/dist/parsers/__tests__/long-tail.golden.test.d.ts +0 -2
- package/dist/parsers/__tests__/long-tail.golden.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/long-tail.golden.test.js +0 -115
- package/dist/parsers/__tests__/long-tail.golden.test.js.map +0 -1
- package/dist/parsers/__tests__/lwc-bindings.test.d.ts +0 -2
- package/dist/parsers/__tests__/lwc-bindings.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/lwc-bindings.test.js +0 -149
- package/dist/parsers/__tests__/lwc-bindings.test.js.map +0 -1
- package/dist/parsers/__tests__/lwc.golden.test.d.ts +0 -2
- package/dist/parsers/__tests__/lwc.golden.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/lwc.golden.test.js +0 -16
- package/dist/parsers/__tests__/lwc.golden.test.js.map +0 -1
- package/dist/parsers/__tests__/mcd-gap-fills.test.d.ts +0 -2
- package/dist/parsers/__tests__/mcd-gap-fills.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/mcd-gap-fills.test.js +0 -75
- package/dist/parsers/__tests__/mcd-gap-fills.test.js.map +0 -1
- package/dist/parsers/__tests__/object-inline-fields.test.d.ts +0 -2
- package/dist/parsers/__tests__/object-inline-fields.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/object-inline-fields.test.js +0 -115
- package/dist/parsers/__tests__/object-inline-fields.test.js.map +0 -1
- package/dist/parsers/__tests__/object.golden.test.d.ts +0 -2
- package/dist/parsers/__tests__/object.golden.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/object.golden.test.js +0 -33
- package/dist/parsers/__tests__/object.golden.test.js.map +0 -1
- package/dist/parsers/__tests__/omnistudio-overlap.test.d.ts +0 -2
- package/dist/parsers/__tests__/omnistudio-overlap.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/omnistudio-overlap.test.js +0 -163
- package/dist/parsers/__tests__/omnistudio-overlap.test.js.map +0 -1
- package/dist/parsers/__tests__/omnistudio.golden.test.d.ts +0 -2
- package/dist/parsers/__tests__/omnistudio.golden.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/omnistudio.golden.test.js +0 -53
- package/dist/parsers/__tests__/omnistudio.golden.test.js.map +0 -1
- package/dist/parsers/__tests__/reflection-walker.test.d.ts +0 -2
- package/dist/parsers/__tests__/reflection-walker.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/reflection-walker.test.js +0 -162
- package/dist/parsers/__tests__/reflection-walker.test.js.map +0 -1
- package/dist/parsers/__tests__/registry.test.d.ts +0 -2
- package/dist/parsers/__tests__/registry.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/registry.test.js +0 -23
- package/dist/parsers/__tests__/registry.test.js.map +0 -1
- package/dist/parsers/__tests__/security.golden.test.d.ts +0 -2
- package/dist/parsers/__tests__/security.golden.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/security.golden.test.js +0 -31
- package/dist/parsers/__tests__/security.golden.test.js.map +0 -1
- package/dist/parsers/__tests__/vlocity.golden.test.d.ts +0 -2
- package/dist/parsers/__tests__/vlocity.golden.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/vlocity.golden.test.js +0 -105
- package/dist/parsers/__tests__/vlocity.golden.test.js.map +0 -1
- package/dist/parsers/__tests__/worker-pool.test.d.ts +0 -2
- package/dist/parsers/__tests__/worker-pool.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/worker-pool.test.js +0 -48
- package/dist/parsers/__tests__/worker-pool.test.js.map +0 -1
- package/dist/parsers/apex/__tests__/arity-resolver.test.d.ts +0 -2
- package/dist/parsers/apex/__tests__/arity-resolver.test.d.ts.map +0 -1
- package/dist/parsers/apex/__tests__/arity-resolver.test.js +0 -135
- package/dist/parsers/apex/__tests__/arity-resolver.test.js.map +0 -1
- package/dist/parsers/apex/__tests__/ast-extractor.test.d.ts +0 -2
- package/dist/parsers/apex/__tests__/ast-extractor.test.d.ts.map +0 -1
- package/dist/parsers/apex/__tests__/ast-extractor.test.js +0 -206
- package/dist/parsers/apex/__tests__/ast-extractor.test.js.map +0 -1
- package/dist/parsers/apex/__tests__/empty-body.test.d.ts +0 -2
- package/dist/parsers/apex/__tests__/empty-body.test.d.ts.map +0 -1
- package/dist/parsers/apex/__tests__/empty-body.test.js +0 -40
- package/dist/parsers/apex/__tests__/empty-body.test.js.map +0 -1
- package/dist/parsers/apex/__tests__/is-test-attribute.test.d.ts +0 -2
- package/dist/parsers/apex/__tests__/is-test-attribute.test.d.ts.map +0 -1
- package/dist/parsers/apex/__tests__/is-test-attribute.test.js +0 -56
- package/dist/parsers/apex/__tests__/is-test-attribute.test.js.map +0 -1
- package/dist/parsers/apex/__tests__/regex-arg-count.test.d.ts +0 -2
- package/dist/parsers/apex/__tests__/regex-arg-count.test.d.ts.map +0 -1
- package/dist/parsers/apex/__tests__/regex-arg-count.test.js +0 -82
- package/dist/parsers/apex/__tests__/regex-arg-count.test.js.map +0 -1
- package/dist/parsers/apex/__tests__/regex-inloop.test.d.ts +0 -2
- package/dist/parsers/apex/__tests__/regex-inloop.test.d.ts.map +0 -1
- package/dist/parsers/apex/__tests__/regex-inloop.test.js +0 -70
- package/dist/parsers/apex/__tests__/regex-inloop.test.js.map +0 -1
- package/dist/parsers/flow/__tests__/invocable-resolver.test.d.ts +0 -2
- package/dist/parsers/flow/__tests__/invocable-resolver.test.d.ts.map +0 -1
- package/dist/parsers/flow/__tests__/invocable-resolver.test.js +0 -91
- package/dist/parsers/flow/__tests__/invocable-resolver.test.js.map +0 -1
- package/dist/parsers/rules/__tests__/applies-to.test.d.ts +0 -2
- package/dist/parsers/rules/__tests__/applies-to.test.d.ts.map +0 -1
- package/dist/parsers/rules/__tests__/applies-to.test.js +0 -31
- package/dist/parsers/rules/__tests__/applies-to.test.js.map +0 -1
- package/dist/parsers/rules/__tests__/applies-to.test.ts +0 -40
- package/dist/parsers/rules/__tests__/engine.test.d.ts +0 -2
- package/dist/parsers/rules/__tests__/engine.test.d.ts.map +0 -1
- package/dist/parsers/rules/__tests__/engine.test.js +0 -127
- package/dist/parsers/rules/__tests__/engine.test.js.map +0 -1
- package/dist/parsers/rules/__tests__/engine.test.ts +0 -138
- package/dist/parsers/rules/__tests__/loader.test.d.ts +0 -2
- package/dist/parsers/rules/__tests__/loader.test.d.ts.map +0 -1
- package/dist/parsers/rules/__tests__/loader.test.js +0 -31
- package/dist/parsers/rules/__tests__/loader.test.js.map +0 -1
- package/dist/parsers/rules/__tests__/loader.test.ts +0 -37
- package/dist/parsers/rules/__tests__/schema.test.d.ts +0 -2
- package/dist/parsers/rules/__tests__/schema.test.d.ts.map +0 -1
- package/dist/parsers/rules/__tests__/schema.test.js +0 -33
- package/dist/parsers/rules/__tests__/schema.test.js.map +0 -1
- package/dist/parsers/rules/__tests__/schema.test.ts +0 -37
- package/dist/parsers/rules/__tests__/selectors.test.d.ts +0 -2
- package/dist/parsers/rules/__tests__/selectors.test.d.ts.map +0 -1
- package/dist/parsers/rules/__tests__/selectors.test.js +0 -51
- package/dist/parsers/rules/__tests__/selectors.test.js.map +0 -1
- package/dist/parsers/rules/__tests__/selectors.test.ts +0 -62
- package/dist/render/__tests__/sarif.test.d.ts +0 -2
- package/dist/render/__tests__/sarif.test.d.ts.map +0 -1
- package/dist/render/__tests__/sarif.test.js +0 -126
- package/dist/render/__tests__/sarif.test.js.map +0 -1
- package/dist/render/mermaid/__tests__/mermaid.test.d.ts +0 -2
- package/dist/render/mermaid/__tests__/mermaid.test.d.ts.map +0 -1
- package/dist/render/mermaid/__tests__/mermaid.test.js +0 -74
- package/dist/render/mermaid/__tests__/mermaid.test.js.map +0 -1
- package/dist/storage/__tests__/analysis-tables-migration.test.d.ts +0 -2
- package/dist/storage/__tests__/analysis-tables-migration.test.d.ts.map +0 -1
- package/dist/storage/__tests__/analysis-tables-migration.test.js +0 -44
- package/dist/storage/__tests__/analysis-tables-migration.test.js.map +0 -1
- package/dist/storage/__tests__/graph-store.test.d.ts +0 -2
- package/dist/storage/__tests__/graph-store.test.d.ts.map +0 -1
- package/dist/storage/__tests__/graph-store.test.js +0 -145
- package/dist/storage/__tests__/graph-store.test.js.map +0 -1
- package/dist/storage/__tests__/identifier.test.d.ts +0 -2
- package/dist/storage/__tests__/identifier.test.d.ts.map +0 -1
- package/dist/storage/__tests__/identifier.test.js +0 -35
- package/dist/storage/__tests__/identifier.test.js.map +0 -1
- package/dist/storage/__tests__/load-better-sqlite3.test.d.ts +0 -2
- package/dist/storage/__tests__/load-better-sqlite3.test.d.ts.map +0 -1
- package/dist/storage/__tests__/load-better-sqlite3.test.js +0 -65
- package/dist/storage/__tests__/load-better-sqlite3.test.js.map +0 -1
- package/dist/storage/__tests__/migrations.test.d.ts +0 -2
- package/dist/storage/__tests__/migrations.test.d.ts.map +0 -1
- package/dist/storage/__tests__/migrations.test.js +0 -93
- package/dist/storage/__tests__/migrations.test.js.map +0 -1
- package/dist/storage/__tests__/perf-50k.test.d.ts +0 -2
- package/dist/storage/__tests__/perf-50k.test.d.ts.map +0 -1
- package/dist/storage/__tests__/perf-50k.test.js +0 -67
- package/dist/storage/__tests__/perf-50k.test.js.map +0 -1
- package/dist/storage/__tests__/rename-stability.test.d.ts +0 -2
- package/dist/storage/__tests__/rename-stability.test.d.ts.map +0 -1
- package/dist/storage/__tests__/rename-stability.test.js +0 -135
- package/dist/storage/__tests__/rename-stability.test.js.map +0 -1
- package/dist/storage/__tests__/snapshot-store.test.d.ts +0 -2
- package/dist/storage/__tests__/snapshot-store.test.d.ts.map +0 -1
- package/dist/storage/__tests__/snapshot-store.test.js +0 -117
- package/dist/storage/__tests__/snapshot-store.test.js.map +0 -1
- package/dist/storage/__tests__/snippets.test.d.ts +0 -2
- package/dist/storage/__tests__/snippets.test.d.ts.map +0 -1
- package/dist/storage/__tests__/snippets.test.js +0 -88
- package/dist/storage/__tests__/snippets.test.js.map +0 -1
- package/dist/storage/__tests__/sync-generation.test.d.ts +0 -2
- package/dist/storage/__tests__/sync-generation.test.d.ts.map +0 -1
- package/dist/storage/__tests__/sync-generation.test.js +0 -159
- package/dist/storage/__tests__/sync-generation.test.js.map +0 -1
- package/dist/storage/__tests__/vector-store-load-failure.test.d.ts +0 -2
- package/dist/storage/__tests__/vector-store-load-failure.test.d.ts.map +0 -1
- package/dist/storage/__tests__/vector-store-load-failure.test.js +0 -42
- package/dist/storage/__tests__/vector-store-load-failure.test.js.map +0 -1
- package/dist/storage/__tests__/vector-store.test.d.ts +0 -2
- package/dist/storage/__tests__/vector-store.test.d.ts.map +0 -1
- package/dist/storage/__tests__/vector-store.test.js +0 -87
- package/dist/storage/__tests__/vector-store.test.js.map +0 -1
- package/dist/telemetry/__tests__/event-schema.test.d.ts +0 -2
- package/dist/telemetry/__tests__/event-schema.test.d.ts.map +0 -1
- package/dist/telemetry/__tests__/event-schema.test.js +0 -56
- package/dist/telemetry/__tests__/event-schema.test.js.map +0 -1
- package/dist/telemetry/__tests__/machine-id.test.d.ts +0 -2
- package/dist/telemetry/__tests__/machine-id.test.d.ts.map +0 -1
- package/dist/telemetry/__tests__/machine-id.test.js +0 -33
- package/dist/telemetry/__tests__/machine-id.test.js.map +0 -1
- package/dist/telemetry/__tests__/pipeline.test.d.ts +0 -2
- package/dist/telemetry/__tests__/pipeline.test.d.ts.map +0 -1
- package/dist/telemetry/__tests__/pipeline.test.js +0 -68
- package/dist/telemetry/__tests__/pipeline.test.js.map +0 -1
- package/dist/telemetry/__tests__/sanitizer.test.d.ts +0 -2
- package/dist/telemetry/__tests__/sanitizer.test.d.ts.map +0 -1
- package/dist/telemetry/__tests__/sanitizer.test.js +0 -143
- package/dist/telemetry/__tests__/sanitizer.test.js.map +0 -1
|
@@ -1,10 +1,91 @@
|
|
|
1
|
-
import { mkdirSync } from "node:fs";
|
|
1
|
+
import { mkdirSync, statSync } from "node:fs";
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import { StorageError, asOrgId, asQualifiedName, asSha256 } from "@ryanstark24/sfgraph-shared";
|
|
4
4
|
import Database from "better-sqlite3";
|
|
5
|
+
import { splitIdentifierWords } from "../../embedding/tokenize.js";
|
|
6
|
+
import { isThisHost, ownStartMs, ownerIsStale, thisHost } from "../../util/process-owner.js";
|
|
5
7
|
import { edgeTableName, nodeTableName, validateLabel, validateRelType } from "../identifier.js";
|
|
6
8
|
import { wrapAbiError } from "./load-better-sqlite3.js";
|
|
7
9
|
import { MIGRATIONS, MigrationRunner } from "./migrations.js";
|
|
10
|
+
/**
|
|
11
|
+
* Plain (non-vec0) tables holding derived data keyed by (org_id, qualified_name).
|
|
12
|
+
* Purged in deleteNode so a removed node leaves no orphan rows. The vec0 vector
|
|
13
|
+
* table is NOT here — it needs the sqlite-vec extension and is purged via the
|
|
14
|
+
* VectorStore. Snapshot tables are intentionally excluded (point-in-time
|
|
15
|
+
* history must survive a node's deletion).
|
|
16
|
+
*/
|
|
17
|
+
const DERIVED_QNAME_TABLES = [
|
|
18
|
+
"_sfgraph_snippets",
|
|
19
|
+
"_sfgraph_governor_risks",
|
|
20
|
+
"_sfgraph_findings",
|
|
21
|
+
"_sfgraph_dead_code_scores",
|
|
22
|
+
"_sfgraph_test_coverage",
|
|
23
|
+
];
|
|
24
|
+
// _sfgraph_node_fts is also derived per-qname, but its key columns are FTS5
|
|
25
|
+
// UNINDEXED (a WHERE on them scans the whole table), so it is purged by rowid
|
|
26
|
+
// through _sfgraph_node_fts_rowid instead — see deleteNodeFts.
|
|
27
|
+
/** How long a write waits for another connection's lock (rel-05). */
|
|
28
|
+
export const BUSY_TIMEOUT_MS = 5000;
|
|
29
|
+
/**
|
|
30
|
+
* Page cache per connection, in KiB (`cache_size = -N`). Was 200 MB: whole-graph
|
|
31
|
+
* phases (resolvers, audit, deadcode) fill it on a large org DB (1.4 GB), and
|
|
32
|
+
* the pages stay resident. 64 MB kept a large Vlocity repo rebuild's wall time
|
|
33
|
+
* unchanged (3 interleaved runs each: 55.0 s at 200 MB vs 53.1 s).
|
|
34
|
+
*/
|
|
35
|
+
export const SQLITE_CACHE_SIZE_KIB = 64_000;
|
|
36
|
+
/** Memory-mapped I/O window per connection, in bytes (was 256 MiB; same measurement). */
|
|
37
|
+
export const SQLITE_MMAP_SIZE_BYTES = 64 * 1024 * 1024;
|
|
38
|
+
/** Members per bulk transaction (beginBulk/endBulk). */
|
|
39
|
+
export const BULK_TXN_MEMBERS = 2000;
|
|
40
|
+
/**
|
|
41
|
+
* Live-graph write batches (beginBatch): members per transaction, and the
|
|
42
|
+
* longest one transaction holds the write lock. A non-rebuild ingest writes
|
|
43
|
+
* into a graph that MCP servers may be reading (and, rarely, writing): readers
|
|
44
|
+
* are never blocked in WAL mode, and another writer waits at most about this
|
|
45
|
+
* long, far inside BUSY_TIMEOUT_MS.
|
|
46
|
+
*/
|
|
47
|
+
export const BATCH_TXN_MEMBERS = 250;
|
|
48
|
+
export const BATCH_TXN_MAX_MS = 200;
|
|
49
|
+
/** Edge tables per UNION ALL statement (SQLite's default compound-SELECT cap is 500). */
|
|
50
|
+
const MAX_UNION_ARMS = 200;
|
|
51
|
+
/** Unit separator; cannot occur in a qname or org id. */
|
|
52
|
+
const KSEP = String.fromCharCode(31);
|
|
53
|
+
/** Canonical secondary-index DDL of a node / edge table. The single source for
|
|
54
|
+
* ensureNodeTable / ensureEdgeTable and the post-crash heal in init(), so a
|
|
55
|
+
* healed index has byte-identical sqlite_master SQL (SQLite stores it without
|
|
56
|
+
* the IF NOT EXISTS and the trailing semicolon). */
|
|
57
|
+
function nodeIndexDdl(tbl) {
|
|
58
|
+
return `CREATE INDEX IF NOT EXISTS ${tbl}_org ON ${tbl}(org_id);`;
|
|
59
|
+
}
|
|
60
|
+
function edgeIndexDdl(tbl) {
|
|
61
|
+
return `CREATE INDEX IF NOT EXISTS ${tbl}_rev ON ${tbl}(org_id, dst_qname);`;
|
|
62
|
+
}
|
|
63
|
+
/** Durable marker of an open bulk window (one row). Written before any DROP
|
|
64
|
+
* INDEX and cleared only once endBulk has rebuilt the indexes, so a process
|
|
65
|
+
* that dies inside the window is detected and healed by the next init(). */
|
|
66
|
+
const BULK_MARKER_DDL = `CREATE TABLE IF NOT EXISTS _sfgraph_bulk_window (
|
|
67
|
+
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
68
|
+
pid INTEGER NOT NULL,
|
|
69
|
+
started_at INTEGER NOT NULL,
|
|
70
|
+
host TEXT
|
|
71
|
+
)`;
|
|
72
|
+
function chunks(xs, n) {
|
|
73
|
+
const out = [];
|
|
74
|
+
for (let i = 0; i < xs.length; i += n)
|
|
75
|
+
out.push(xs.slice(i, i + n));
|
|
76
|
+
return out;
|
|
77
|
+
}
|
|
78
|
+
/** ASCII-only lowercase — exactly what SQLite's NOCASE collation folds. */
|
|
79
|
+
function nocaseKey(s) {
|
|
80
|
+
// toLowerCase is exactly the ASCII fold on an ASCII-only string (the common
|
|
81
|
+
// case, and ~10x cheaper than a replace callback); non-ASCII letters must
|
|
82
|
+
// stay as they are, so those strings fold A-Z only.
|
|
83
|
+
for (let i = 0; i < s.length; i++) {
|
|
84
|
+
if (s.charCodeAt(i) > 0x7f)
|
|
85
|
+
return s.replace(/[A-Z]/g, (c) => c.toLowerCase());
|
|
86
|
+
}
|
|
87
|
+
return s.toLowerCase();
|
|
88
|
+
}
|
|
8
89
|
function rowToNode(row, label) {
|
|
9
90
|
return {
|
|
10
91
|
orgId: asOrgId(row.org_id),
|
|
@@ -28,18 +109,115 @@ function rowToEdge(row, relType) {
|
|
|
28
109
|
lastSeenAt: row.last_seen_at,
|
|
29
110
|
};
|
|
30
111
|
}
|
|
112
|
+
/**
|
|
113
|
+
* rel-02: `ownerIsStale` may spawn `ps` to rule out pid reuse; readers call
|
|
114
|
+
* getSyncStatus on every MCP tool call, so verdicts are memoised briefly per
|
|
115
|
+
* recorded owner. A dead owner never comes back, so a stale verdict is final.
|
|
116
|
+
*/
|
|
117
|
+
const SYNC_OWNER_TTL_MS = 30_000;
|
|
118
|
+
const syncOwnerVerdicts = new Map();
|
|
119
|
+
function syncOwnerIsStale(pid, host, startMs) {
|
|
120
|
+
if (pid === process.pid && isThisHost(host))
|
|
121
|
+
return false;
|
|
122
|
+
const key = `${host ?? ""}|${pid}|${startMs ?? ""}`;
|
|
123
|
+
const now = Date.now();
|
|
124
|
+
const hit = syncOwnerVerdicts.get(key);
|
|
125
|
+
if (hit && (hit.stale || now - hit.at < SYNC_OWNER_TTL_MS))
|
|
126
|
+
return hit.stale;
|
|
127
|
+
const stale = ownerIsStale({ pid, host, procStartMs: startMs });
|
|
128
|
+
if (syncOwnerVerdicts.size > 256)
|
|
129
|
+
syncOwnerVerdicts.clear();
|
|
130
|
+
syncOwnerVerdicts.set(key, { stale, at: now });
|
|
131
|
+
return stale;
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* review R1-1: whether the graph file at `dbPath` is a finished build that a
|
|
135
|
+
* reader may switch to. `ingest --rebuild` creates the new file at dbPath and
|
|
136
|
+
* fills it for minutes (indexes dropped until endBulk), so a reader that sees
|
|
137
|
+
* a new inode must keep its old connection until the new file:
|
|
138
|
+
* - has no bulk-load window whose owner is alive,
|
|
139
|
+
* - has an `_sfgraph_orgs` row (for `orgId`, or every row when omitted), and
|
|
140
|
+
* each such row has a completed sync (`last_synced_at`) and is not in
|
|
141
|
+
* progress with a live owner (same rule as getSyncStatus, rel-02).
|
|
142
|
+
* Opens the file read-only and never writes.
|
|
143
|
+
*/
|
|
144
|
+
export function graphFileReadiness(dbPath, orgId) {
|
|
145
|
+
let db;
|
|
146
|
+
try {
|
|
147
|
+
db = new Database(dbPath, { readonly: true, fileMustExist: true });
|
|
148
|
+
}
|
|
149
|
+
catch {
|
|
150
|
+
return { ready: false, reason: "the file cannot be opened" };
|
|
151
|
+
}
|
|
152
|
+
try {
|
|
153
|
+
const has = (t) => db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(t) !==
|
|
154
|
+
undefined;
|
|
155
|
+
if (has("_sfgraph_bulk_window")) {
|
|
156
|
+
const m = db.prepare("SELECT * FROM _sfgraph_bulk_window WHERE id = 1").get();
|
|
157
|
+
if (m) {
|
|
158
|
+
const marker = { pid: m.pid, host: m.host ?? null, takenAtMs: m.started_at ?? null };
|
|
159
|
+
const ours = m.pid === process.pid && isThisHost(marker.host);
|
|
160
|
+
if (ours || !ownerIsStale(marker))
|
|
161
|
+
return { ready: false, reason: "a bulk load is running" };
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
if (!has("_sfgraph_orgs"))
|
|
165
|
+
return { ready: false, reason: "no _sfgraph_orgs table" };
|
|
166
|
+
const rows = (orgId === undefined
|
|
167
|
+
? db.prepare("SELECT * FROM _sfgraph_orgs").all()
|
|
168
|
+
: db.prepare("SELECT * FROM _sfgraph_orgs WHERE id = ?").all(orgId));
|
|
169
|
+
if (rows.length === 0)
|
|
170
|
+
return { ready: false, reason: "no org row yet" };
|
|
171
|
+
for (const r of rows) {
|
|
172
|
+
if (r.last_synced_at == null)
|
|
173
|
+
return { ready: false, reason: "no completed sync" };
|
|
174
|
+
if (r.sync_in_progress === 1 &&
|
|
175
|
+
(r.sync_owner_pid == null ||
|
|
176
|
+
!syncOwnerIsStale(r.sync_owner_pid, r.sync_owner_host ?? null, r.sync_owner_start_ms ?? null))) {
|
|
177
|
+
return { ready: false, reason: "a sync is in progress" };
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
return { ready: true };
|
|
181
|
+
}
|
|
182
|
+
catch {
|
|
183
|
+
return { ready: false, reason: "the file is not readable yet" };
|
|
184
|
+
}
|
|
185
|
+
finally {
|
|
186
|
+
try {
|
|
187
|
+
db.close();
|
|
188
|
+
}
|
|
189
|
+
catch {
|
|
190
|
+
/* ignore */
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
}
|
|
31
194
|
export class SqliteGraphStore {
|
|
32
|
-
|
|
195
|
+
conn;
|
|
196
|
+
/** Connections replaced after a file swap; closed with the store (review I2). */
|
|
197
|
+
retired = [];
|
|
198
|
+
/** `dev:ino` of dbPath for the open connection; null when not file-backed. */
|
|
199
|
+
fileId = null;
|
|
200
|
+
lastFileCheck = 0;
|
|
33
201
|
ownsDb;
|
|
34
202
|
opts;
|
|
35
203
|
nodeLabelCache = new Map();
|
|
36
204
|
edgeRelCache = new Map();
|
|
205
|
+
/** `PRAGMA data_version` when the caches were last loaded (rel-01). */
|
|
206
|
+
dataVersion = -1;
|
|
37
207
|
initialized = false;
|
|
38
208
|
closed = false;
|
|
209
|
+
bulk = null;
|
|
210
|
+
batch = null;
|
|
211
|
+
/** edgesAllTables' SQL per column, for the current edge tables. */
|
|
212
|
+
unionEdgeQueries = {};
|
|
213
|
+
/** priorOutgoingEdges' SQL for the current edge tables. */
|
|
214
|
+
priorEdgesQuery = null;
|
|
215
|
+
stmtCache = new Map();
|
|
216
|
+
ftsUpsertTx = null;
|
|
39
217
|
constructor(opts) {
|
|
40
218
|
this.opts = opts;
|
|
41
219
|
if (opts.db) {
|
|
42
|
-
this.
|
|
220
|
+
this.conn = opts.db;
|
|
43
221
|
this.ownsDb = false;
|
|
44
222
|
}
|
|
45
223
|
else {
|
|
@@ -49,7 +227,7 @@ export class SqliteGraphStore {
|
|
|
49
227
|
mkdirSync(dir, { recursive: true });
|
|
50
228
|
}
|
|
51
229
|
try {
|
|
52
|
-
this.
|
|
230
|
+
this.conn = new Database(opts.dbPath);
|
|
53
231
|
}
|
|
54
232
|
catch (e) {
|
|
55
233
|
const wrapped = wrapAbiError(e);
|
|
@@ -60,15 +238,29 @@ export class SqliteGraphStore {
|
|
|
60
238
|
this.ownsDb = true;
|
|
61
239
|
}
|
|
62
240
|
}
|
|
241
|
+
get db() {
|
|
242
|
+
return this.conn;
|
|
243
|
+
}
|
|
63
244
|
async init() {
|
|
64
245
|
if (this.initialized)
|
|
65
246
|
return;
|
|
247
|
+
this.fileId = this.readFileId();
|
|
248
|
+
this.setupConnection();
|
|
249
|
+
this.loadCaches();
|
|
250
|
+
this.healBulkWindow();
|
|
251
|
+
this.initialized = true;
|
|
252
|
+
}
|
|
253
|
+
/** Pragmas + pending migrations for a freshly opened connection. */
|
|
254
|
+
setupConnection() {
|
|
255
|
+
// rel-05: explicit, not better-sqlite3's implicit default. A write that
|
|
256
|
+
// still finds the lock held after this throws SQLITE_BUSY.
|
|
257
|
+
this.db.pragma(`busy_timeout = ${BUSY_TIMEOUT_MS}`);
|
|
66
258
|
this.db.pragma("journal_mode = WAL");
|
|
67
259
|
this.db.pragma("synchronous = NORMAL");
|
|
68
260
|
this.db.pragma("foreign_keys = ON");
|
|
69
261
|
this.db.pragma("temp_store = MEMORY");
|
|
70
|
-
this.db.pragma(
|
|
71
|
-
this.db.pragma(
|
|
262
|
+
this.db.pragma(`mmap_size = ${SQLITE_MMAP_SIZE_BYTES}`);
|
|
263
|
+
this.db.pragma(`cache_size = -${SQLITE_CACHE_SIZE_KIB}`);
|
|
72
264
|
// Smaller WAL auto-checkpoint threshold (default 1000 pages = ~4MB).
|
|
73
265
|
// With large ingests producing 10-20K inserts during the object phase,
|
|
74
266
|
// the WAL hits the default threshold right around the 30-second mark
|
|
@@ -88,8 +280,74 @@ export class SqliteGraphStore {
|
|
|
88
280
|
retainBackups: this.opts.retainBackups ?? 5,
|
|
89
281
|
});
|
|
90
282
|
runner.applyPending();
|
|
91
|
-
|
|
92
|
-
|
|
283
|
+
}
|
|
284
|
+
readFileId() {
|
|
285
|
+
if (!this.ownsDb || this.opts.dbPath === ":memory:")
|
|
286
|
+
return null;
|
|
287
|
+
try {
|
|
288
|
+
const st = statSync(this.opts.dbPath);
|
|
289
|
+
return `${st.dev}:${st.ino}`;
|
|
290
|
+
}
|
|
291
|
+
catch {
|
|
292
|
+
return null;
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
/**
|
|
296
|
+
* review I2: `ingest --rebuild` renames dbPath away and builds a new file.
|
|
297
|
+
* This connection still holds the old inode, whose data_version never
|
|
298
|
+
* changes again, so compare dbPath's dev/ino and reopen when it differs.
|
|
299
|
+
* The old connection stays open until close(): other holders (shared
|
|
300
|
+
* snapshot/vector stores) may still have statements on it.
|
|
301
|
+
*/
|
|
302
|
+
checkFileSwap() {
|
|
303
|
+
if (this.fileId === null)
|
|
304
|
+
return;
|
|
305
|
+
const every = this.opts.fileCheckIntervalMs ?? 1000;
|
|
306
|
+
if (every < 0)
|
|
307
|
+
return;
|
|
308
|
+
const now = Date.now();
|
|
309
|
+
if (every > 0 && now - this.lastFileCheck < every)
|
|
310
|
+
return;
|
|
311
|
+
this.lastFileCheck = now;
|
|
312
|
+
if (this.bulk || this.batch || this.conn.inTransaction)
|
|
313
|
+
return;
|
|
314
|
+
const id = this.readFileId();
|
|
315
|
+
if (id === null || id === this.fileId)
|
|
316
|
+
return;
|
|
317
|
+
// review R1-1: keep serving the old (complete) graph until the new file's
|
|
318
|
+
// build has finished; the old inode stays readable after rename/unlink.
|
|
319
|
+
if (!graphFileReadiness(this.opts.dbPath).ready)
|
|
320
|
+
return;
|
|
321
|
+
let next;
|
|
322
|
+
try {
|
|
323
|
+
next = new Database(this.opts.dbPath);
|
|
324
|
+
}
|
|
325
|
+
catch {
|
|
326
|
+
return; // mid-swap; retry on a later read
|
|
327
|
+
}
|
|
328
|
+
const prev = this.conn;
|
|
329
|
+
this.conn = next;
|
|
330
|
+
this.stmtCache.clear();
|
|
331
|
+
this.ftsUpsertTx = null;
|
|
332
|
+
try {
|
|
333
|
+
this.setupConnection();
|
|
334
|
+
this.loadCaches();
|
|
335
|
+
}
|
|
336
|
+
catch {
|
|
337
|
+
// e.g. the rebuild still holds the write lock: keep the old graph for now.
|
|
338
|
+
this.conn = prev;
|
|
339
|
+
this.stmtCache.clear();
|
|
340
|
+
try {
|
|
341
|
+
next.close();
|
|
342
|
+
}
|
|
343
|
+
catch {
|
|
344
|
+
/* ignore */
|
|
345
|
+
}
|
|
346
|
+
this.loadCaches();
|
|
347
|
+
return;
|
|
348
|
+
}
|
|
349
|
+
this.retired.push(prev);
|
|
350
|
+
this.fileId = id;
|
|
93
351
|
}
|
|
94
352
|
/**
|
|
95
353
|
* Flush WAL to the main DB file with a PASSIVE checkpoint. Cheap (no
|
|
@@ -114,6 +372,22 @@ export class SqliteGraphStore {
|
|
|
114
372
|
async close() {
|
|
115
373
|
if (this.closed)
|
|
116
374
|
return;
|
|
375
|
+
if (this.bulk) {
|
|
376
|
+
try {
|
|
377
|
+
this.endBulk();
|
|
378
|
+
}
|
|
379
|
+
catch {
|
|
380
|
+
/* best-effort; the next init() heals any missing indexes */
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
if (this.batch) {
|
|
384
|
+
try {
|
|
385
|
+
this.endBatch();
|
|
386
|
+
}
|
|
387
|
+
catch {
|
|
388
|
+
/* best-effort */
|
|
389
|
+
}
|
|
390
|
+
}
|
|
117
391
|
this.closed = true;
|
|
118
392
|
if (this.ownsDb) {
|
|
119
393
|
try {
|
|
@@ -131,29 +405,75 @@ export class SqliteGraphStore {
|
|
|
131
405
|
catch {
|
|
132
406
|
// already closed by another path — idempotent
|
|
133
407
|
}
|
|
408
|
+
for (const old of this.retired.splice(0)) {
|
|
409
|
+
try {
|
|
410
|
+
old.close();
|
|
411
|
+
}
|
|
412
|
+
catch {
|
|
413
|
+
/* ignore */
|
|
414
|
+
}
|
|
415
|
+
}
|
|
134
416
|
}
|
|
135
417
|
}
|
|
136
418
|
loadCaches() {
|
|
137
|
-
const labels = this.
|
|
138
|
-
|
|
139
|
-
.all();
|
|
419
|
+
const labels = this.stmt("SELECT label, table_name FROM _sfgraph_node_labels").all();
|
|
420
|
+
const nodeLabels = new Map();
|
|
140
421
|
for (const row of labels)
|
|
141
|
-
|
|
142
|
-
const edges = this.
|
|
143
|
-
|
|
144
|
-
.all();
|
|
422
|
+
nodeLabels.set(row.label, row.table_name);
|
|
423
|
+
const edges = this.stmt("SELECT rel_type, table_name FROM _sfgraph_edge_types").all();
|
|
424
|
+
const edgeRels = new Map();
|
|
145
425
|
for (const row of edges)
|
|
146
|
-
|
|
426
|
+
edgeRels.set(row.rel_type, row.table_name);
|
|
427
|
+
this.nodeLabelCache = nodeLabels;
|
|
428
|
+
this.edgeRelCache = edgeRels;
|
|
429
|
+
this.dataVersion = this.readDataVersion();
|
|
430
|
+
}
|
|
431
|
+
readDataVersion() {
|
|
432
|
+
return this.stmt("PRAGMA data_version").get().data_version;
|
|
433
|
+
}
|
|
434
|
+
/**
|
|
435
|
+
* rel-01: the label / rel-type registries can grow under a long-lived
|
|
436
|
+
* reader (an ingest in another process creates a new node or edge table).
|
|
437
|
+
* `PRAGMA data_version` changes only when ANOTHER connection commits, so
|
|
438
|
+
* this is one cheap read per call and a reload only after a foreign write.
|
|
439
|
+
* Writes through this connection keep the caches current themselves.
|
|
440
|
+
*/
|
|
441
|
+
syncCaches() {
|
|
442
|
+
if (this.closed)
|
|
443
|
+
return;
|
|
444
|
+
this.checkFileSwap();
|
|
445
|
+
if (this.readDataVersion() !== this.dataVersion)
|
|
446
|
+
this.loadCaches();
|
|
447
|
+
}
|
|
448
|
+
/** Cached prepared statement (re-preparing per call cost ~14 s per full ingest). */
|
|
449
|
+
stmt(sql) {
|
|
450
|
+
let st = this.stmtCache.get(sql);
|
|
451
|
+
if (!st) {
|
|
452
|
+
st = this.db.prepare(sql);
|
|
453
|
+
this.stmtCache.set(sql, st);
|
|
454
|
+
}
|
|
455
|
+
return st;
|
|
456
|
+
}
|
|
457
|
+
/** Run a table's secondary-index DDL now, or defer it to endBulk. */
|
|
458
|
+
createIndex(name, table, sql) {
|
|
459
|
+
if (this.bulk) {
|
|
460
|
+
this.bulk.deferredIndexes.push({ name, table, sql });
|
|
461
|
+
return;
|
|
462
|
+
}
|
|
463
|
+
this.db.exec(sql);
|
|
147
464
|
}
|
|
148
465
|
ensureNodeTable(label) {
|
|
149
466
|
const cached = this.nodeLabelCache.get(label);
|
|
150
467
|
if (cached)
|
|
151
468
|
return cached;
|
|
152
469
|
const tbl = nodeTableName(label);
|
|
470
|
+
// qualified_name is COLLATE NOCASE: Salesforce API names are case-insensitive
|
|
471
|
+
// (Account == account), so the PK dedups and all `= ?` lookups fold case while
|
|
472
|
+
// the original display string is preserved. See migration 9.
|
|
153
473
|
this.db.exec(`
|
|
154
474
|
CREATE TABLE IF NOT EXISTS ${tbl} (
|
|
155
475
|
org_id TEXT NOT NULL,
|
|
156
|
-
qualified_name TEXT NOT NULL,
|
|
476
|
+
qualified_name TEXT NOT NULL COLLATE NOCASE,
|
|
157
477
|
attributes TEXT NOT NULL,
|
|
158
478
|
source_hash TEXT NOT NULL,
|
|
159
479
|
first_seen_at INTEGER NOT NULL,
|
|
@@ -161,8 +481,8 @@ export class SqliteGraphStore {
|
|
|
161
481
|
last_modified_at INTEGER NOT NULL,
|
|
162
482
|
PRIMARY KEY(org_id, qualified_name)
|
|
163
483
|
);
|
|
164
|
-
CREATE INDEX IF NOT EXISTS ${tbl}_org ON ${tbl}(org_id);
|
|
165
484
|
`);
|
|
485
|
+
this.createIndex(`${tbl}_org`, tbl, nodeIndexDdl(tbl));
|
|
166
486
|
this.db
|
|
167
487
|
.prepare("INSERT OR IGNORE INTO _sfgraph_node_labels(label, table_name, created_at) VALUES (?, ?, ?)")
|
|
168
488
|
.run(label, tbl, Date.now());
|
|
@@ -174,18 +494,21 @@ export class SqliteGraphStore {
|
|
|
174
494
|
if (cached)
|
|
175
495
|
return cached;
|
|
176
496
|
const tbl = edgeTableName(relType);
|
|
497
|
+
// src/dst qnames COLLATE NOCASE so a reference's case (e.g. SOQL `from
|
|
498
|
+
// account`) matches the definition node's qname and edges dedup/join
|
|
499
|
+
// case-insensitively. See migration 9.
|
|
177
500
|
this.db.exec(`
|
|
178
501
|
CREATE TABLE IF NOT EXISTS ${tbl} (
|
|
179
502
|
org_id TEXT NOT NULL,
|
|
180
|
-
src_qname TEXT NOT NULL,
|
|
181
|
-
dst_qname TEXT NOT NULL,
|
|
503
|
+
src_qname TEXT NOT NULL COLLATE NOCASE,
|
|
504
|
+
dst_qname TEXT NOT NULL COLLATE NOCASE,
|
|
182
505
|
attributes TEXT NOT NULL,
|
|
183
506
|
first_seen_at INTEGER NOT NULL,
|
|
184
507
|
last_seen_at INTEGER NOT NULL,
|
|
185
508
|
PRIMARY KEY(org_id, src_qname, dst_qname)
|
|
186
509
|
);
|
|
187
|
-
CREATE INDEX IF NOT EXISTS ${tbl}_rev ON ${tbl}(org_id, dst_qname);
|
|
188
510
|
`);
|
|
511
|
+
this.createIndex(`${tbl}_rev`, tbl, edgeIndexDdl(tbl));
|
|
189
512
|
this.db
|
|
190
513
|
.prepare("INSERT OR IGNORE INTO _sfgraph_edge_types(rel_type, table_name, created_at) VALUES (?, ?, ?)")
|
|
191
514
|
.run(relType, tbl, Date.now());
|
|
@@ -237,9 +560,14 @@ export class SqliteGraphStore {
|
|
|
237
560
|
if (!Number.isFinite(Date.parse(startedAtIso))) {
|
|
238
561
|
throw new StorageError(`markSyncStarted: invalid ISO timestamp ${JSON.stringify(startedAtIso)}`);
|
|
239
562
|
}
|
|
563
|
+
// rel-02: record the owner so a reader can tell this run from one that
|
|
564
|
+
// was killed without clearing the flag.
|
|
240
565
|
this.db
|
|
241
|
-
.prepare(
|
|
242
|
-
|
|
566
|
+
.prepare(`UPDATE _sfgraph_orgs
|
|
567
|
+
SET sync_in_progress = 1, sync_started_at = ?,
|
|
568
|
+
sync_owner_pid = ?, sync_owner_host = ?, sync_owner_start_ms = ?
|
|
569
|
+
WHERE id = ?`)
|
|
570
|
+
.run(startedAtIso, process.pid, thisHost(), ownStartMs(), orgId);
|
|
243
571
|
}
|
|
244
572
|
markSyncComplete(orgId, completedAtIso) {
|
|
245
573
|
const ts = Date.parse(completedAtIso);
|
|
@@ -254,6 +582,7 @@ export class SqliteGraphStore {
|
|
|
254
582
|
SET sync_generation = sync_generation + 1,
|
|
255
583
|
sync_in_progress = 0,
|
|
256
584
|
sync_started_at = NULL,
|
|
585
|
+
sync_owner_pid = NULL, sync_owner_host = NULL, sync_owner_start_ms = NULL,
|
|
257
586
|
last_synced_at = ?
|
|
258
587
|
WHERE id = ?`)
|
|
259
588
|
.run(ts, orgId);
|
|
@@ -265,12 +594,16 @@ export class SqliteGraphStore {
|
|
|
265
594
|
// version of the graph, so readers polling the counter should not be
|
|
266
595
|
// tricked into thinking data refreshed.
|
|
267
596
|
this.db
|
|
268
|
-
.prepare(
|
|
597
|
+
.prepare(`UPDATE _sfgraph_orgs
|
|
598
|
+
SET sync_in_progress = 0, sync_started_at = NULL,
|
|
599
|
+
sync_owner_pid = NULL, sync_owner_host = NULL, sync_owner_start_ms = NULL
|
|
600
|
+
WHERE id = ?`)
|
|
269
601
|
.run(orgId);
|
|
270
602
|
}
|
|
271
603
|
getSyncStatus(orgId) {
|
|
272
604
|
const row = this.db
|
|
273
|
-
.prepare(`SELECT sync_generation, sync_in_progress, sync_started_at, last_synced_at
|
|
605
|
+
.prepare(`SELECT sync_generation, sync_in_progress, sync_started_at, last_synced_at,
|
|
606
|
+
sync_owner_pid, sync_owner_host, sync_owner_start_ms
|
|
274
607
|
FROM _sfgraph_orgs WHERE id = ?`)
|
|
275
608
|
.get(orgId);
|
|
276
609
|
if (!row) {
|
|
@@ -279,14 +612,23 @@ export class SqliteGraphStore {
|
|
|
279
612
|
// don't have to null-check every field.
|
|
280
613
|
return { generation: 0, in_progress: false, started_at: null, last_sync_at: null };
|
|
281
614
|
}
|
|
615
|
+
// rel-02: a flag whose recorded owner is gone (killed before it could
|
|
616
|
+
// clear it) is reported as not in progress. Report-only: the row is left
|
|
617
|
+
// for the next ingest to overwrite. Legacy rows with no owner are trusted.
|
|
618
|
+
const flagged = (row.sync_in_progress ?? 0) === 1;
|
|
619
|
+
const dead = flagged &&
|
|
620
|
+
row.sync_owner_pid != null &&
|
|
621
|
+
syncOwnerIsStale(row.sync_owner_pid, row.sync_owner_host, row.sync_owner_start_ms);
|
|
622
|
+
const inProgress = flagged && !dead;
|
|
282
623
|
return {
|
|
283
624
|
generation: row.sync_generation ?? 0,
|
|
284
|
-
in_progress:
|
|
285
|
-
started_at: row.sync_started_at ?? null,
|
|
625
|
+
in_progress: inProgress,
|
|
626
|
+
started_at: inProgress ? (row.sync_started_at ?? null) : null,
|
|
286
627
|
last_sync_at: row.last_synced_at ?? null,
|
|
287
628
|
};
|
|
288
629
|
}
|
|
289
630
|
deleteNode(orgId, qname) {
|
|
631
|
+
this.syncCaches();
|
|
290
632
|
const apply = this.db.transaction(() => {
|
|
291
633
|
const idx = this.db
|
|
292
634
|
.prepare("SELECT label FROM _sfgraph_node_index WHERE org_id = ? AND qualified_name = ?")
|
|
@@ -302,15 +644,29 @@ export class SqliteGraphStore {
|
|
|
302
644
|
.prepare("DELETE FROM _sfgraph_node_index WHERE org_id = ? AND qualified_name = ?")
|
|
303
645
|
.run(orgId, qname);
|
|
304
646
|
}
|
|
647
|
+
// Purge derived per-qname rows so a deleted node leaves no orphans that
|
|
648
|
+
// would surface in explain_code (snippet) or the audits (findings /
|
|
649
|
+
// governor risks / dead-code / coverage). Run unconditionally — derived
|
|
650
|
+
// rows can outlive a missing index entry. The vec0 vector lives in a
|
|
651
|
+
// separate extension-loaded connection and is purged by the caller via
|
|
652
|
+
// VectorStore.deleteNodeVector. Each table is keyed (org_id, qualified_name, …).
|
|
653
|
+
for (const t of DERIVED_QNAME_TABLES) {
|
|
654
|
+
this.stmt(`DELETE FROM ${t} WHERE org_id = ? AND qualified_name = ?`).run(orgId, qname);
|
|
655
|
+
}
|
|
656
|
+
this.deleteNodeFts(String(orgId), String(qname));
|
|
305
657
|
});
|
|
306
658
|
apply();
|
|
307
659
|
}
|
|
308
660
|
deleteEdgesFor(orgId, qname) {
|
|
661
|
+
this.syncCaches();
|
|
309
662
|
const apply = this.db.transaction(() => {
|
|
663
|
+
// Two statements, not `src = ? OR dst = ?`: SQLite plans the OR as a scan of the org's
|
|
664
|
+
// whole slice of the table (one `_rev (org_id=?)` search). Split, each side uses an
|
|
665
|
+
// index: the PK `(org_id, src_qname, dst_qname)` and `_rev (org_id, dst_qname)`.
|
|
666
|
+
// About 17x faster on a 300k-edge org; the rows removed are the same.
|
|
310
667
|
for (const tbl of this.edgeRelCache.values()) {
|
|
311
|
-
this.
|
|
312
|
-
|
|
313
|
-
.run(orgId, qname, qname);
|
|
668
|
+
this.stmt(`DELETE FROM ${tbl} WHERE org_id = ? AND src_qname = ?`).run(orgId, qname);
|
|
669
|
+
this.stmt(`DELETE FROM ${tbl} WHERE org_id = ? AND dst_qname = ?`).run(orgId, qname);
|
|
314
670
|
}
|
|
315
671
|
});
|
|
316
672
|
apply();
|
|
@@ -318,6 +674,7 @@ export class SqliteGraphStore {
|
|
|
318
674
|
mergeNodes(facts) {
|
|
319
675
|
if (facts.length === 0)
|
|
320
676
|
return { inserted: 0, updated: 0, unchanged: 0 };
|
|
677
|
+
this.syncCaches();
|
|
321
678
|
const buckets = new Map();
|
|
322
679
|
for (const f of facts) {
|
|
323
680
|
validateLabel(f.label);
|
|
@@ -329,19 +686,38 @@ export class SqliteGraphStore {
|
|
|
329
686
|
let updated = 0;
|
|
330
687
|
let unchanged = 0;
|
|
331
688
|
const apply = this.db.transaction(() => {
|
|
689
|
+
// Look up a node's currently-recorded label (across all per-label tables)
|
|
690
|
+
// so we can detect a label TRANSITION (e.g. ApexMethod → TestMethod when a
|
|
691
|
+
// method gains @isTest, class ↔ interface). On transition the node moves
|
|
692
|
+
// to a new per-label table; the stale old-label row must be deleted, or
|
|
693
|
+
// the detect-deletions sweep sees it as unreferenced and deletes the LIVE
|
|
694
|
+
// node + its edges (deleteNode resolves the table via the index, which now
|
|
695
|
+
// points at the new label). Edges are NOT touched — the qname is unchanged.
|
|
696
|
+
const indexLabelStmt = this.stmt("SELECT label FROM _sfgraph_node_index WHERE org_id = ? AND qualified_name = ?");
|
|
697
|
+
const deleteOldRow = (oldLabel, orgId, qname) => {
|
|
698
|
+
const oldTbl = this.ensureNodeTable(oldLabel);
|
|
699
|
+
this.stmt(`DELETE FROM ${oldTbl} WHERE org_id = ? AND qualified_name = ?`).run(orgId, qname);
|
|
700
|
+
};
|
|
332
701
|
for (const [label, bucket] of buckets) {
|
|
333
702
|
const tbl = this.ensureNodeTable(label);
|
|
334
|
-
const selectStmt = this.
|
|
335
|
-
const insertStmt = this.
|
|
703
|
+
const selectStmt = this.stmt(`SELECT source_hash, first_seen_at FROM ${tbl} WHERE org_id = ? AND qualified_name = ?`);
|
|
704
|
+
const insertStmt = this.stmt(`INSERT INTO ${tbl}(org_id, qualified_name, attributes, source_hash, first_seen_at, last_seen_at, last_modified_at)
|
|
336
705
|
VALUES (?, ?, ?, ?, ?, ?, ?)`);
|
|
337
|
-
const updateChangedStmt = this.
|
|
338
|
-
const updateUnchangedStmt = this.
|
|
339
|
-
const indexStmt = this.
|
|
706
|
+
const updateChangedStmt = this.stmt(`UPDATE ${tbl} SET attributes = ?, source_hash = ?, last_seen_at = ?, last_modified_at = ? WHERE org_id = ? AND qualified_name = ?`);
|
|
707
|
+
const updateUnchangedStmt = this.stmt(`UPDATE ${tbl} SET last_seen_at = ? WHERE org_id = ? AND qualified_name = ?`);
|
|
708
|
+
const indexStmt = this.stmt(`INSERT INTO _sfgraph_node_index(org_id, qualified_name, label) VALUES (?, ?, ?)
|
|
340
709
|
ON CONFLICT(org_id, qualified_name) DO UPDATE SET label=excluded.label`);
|
|
341
710
|
for (const fact of bucket) {
|
|
342
711
|
const existing = selectStmt.get(fact.orgId, fact.qualifiedName);
|
|
343
712
|
const attrsJson = JSON.stringify(fact.attributes);
|
|
344
713
|
if (!existing) {
|
|
714
|
+
// Not present under THIS label. If the index has it under a
|
|
715
|
+
// DIFFERENT label, it's a label transition — purge the stale old row
|
|
716
|
+
// before inserting the new one (see the transaction-level comment).
|
|
717
|
+
const prior = indexLabelStmt.get(fact.orgId, fact.qualifiedName);
|
|
718
|
+
if (prior && prior.label !== label) {
|
|
719
|
+
deleteOldRow(prior.label, String(fact.orgId), String(fact.qualifiedName));
|
|
720
|
+
}
|
|
345
721
|
insertStmt.run(fact.orgId, fact.qualifiedName, attrsJson, fact.sourceHash, fact.firstSeenAt, fact.lastSeenAt, fact.lastModifiedAt);
|
|
346
722
|
indexStmt.run(fact.orgId, fact.qualifiedName, label);
|
|
347
723
|
inserted += 1;
|
|
@@ -360,8 +736,12 @@ export class SqliteGraphStore {
|
|
|
360
736
|
apply();
|
|
361
737
|
return { inserted, updated, unchanged };
|
|
362
738
|
}
|
|
363
|
-
mergeEdges(facts) {
|
|
364
|
-
|
|
739
|
+
mergeEdges(facts, opts) {
|
|
740
|
+
this.syncCaches();
|
|
741
|
+
const extras = opts?.reconcileSources === true && opts.reconcileExtraSrcs?.qnames.length
|
|
742
|
+
? opts.reconcileExtraSrcs
|
|
743
|
+
: undefined;
|
|
744
|
+
if (facts.length === 0 && !extras)
|
|
365
745
|
return { inserted: 0, updated: 0, unchanged: 0 };
|
|
366
746
|
const buckets = new Map();
|
|
367
747
|
for (const f of facts) {
|
|
@@ -370,55 +750,276 @@ export class SqliteGraphStore {
|
|
|
370
750
|
arr.push(f);
|
|
371
751
|
buckets.set(f.relType, arr);
|
|
372
752
|
}
|
|
753
|
+
// Reconciliation bookkeeping: the set of source qnames this batch is
|
|
754
|
+
// authoritative for, plus the exact (org|src|dst|rel) edges it carries, so
|
|
755
|
+
// we can prune any prior outgoing edge from those sources that is no longer
|
|
756
|
+
// emitted. The delimiter is a unit-separator that cannot occur in a qname.
|
|
757
|
+
const reconcile = opts?.reconcileSources === true;
|
|
758
|
+
const RSEP = String.fromCharCode(31);
|
|
759
|
+
// org -> set of source qnames this batch is authoritative for, plus the
|
|
760
|
+
// exact (org|src|dst|rel) edges it carries. Reconciliation prunes prior
|
|
761
|
+
// outgoing edges from those sources that the batch no longer emits.
|
|
762
|
+
const srcByOrg = new Map();
|
|
763
|
+
const batchEdgeKeys = new Set();
|
|
764
|
+
// Bulk fast path: in a fresh-org bulk window, a source this window has not
|
|
765
|
+
// written yet has no prior outgoing edges (the org was empty at beginBulk),
|
|
766
|
+
// so its reconcile probe — one SELECT per edge table per chunk — is skipped.
|
|
767
|
+
// Sources written earlier in the window are still probed (and pruned), so
|
|
768
|
+
// the result is identical to the slow path. Membership is decided BEFORE
|
|
769
|
+
// this batch's inserts are recorded.
|
|
770
|
+
const bulk = this.bulk;
|
|
771
|
+
const freshOrg = bulk?.freshOrg ?? null;
|
|
772
|
+
if (reconcile) {
|
|
773
|
+
for (const f of facts) {
|
|
774
|
+
const org = String(f.orgId);
|
|
775
|
+
const src = String(f.srcQualifiedName);
|
|
776
|
+
// NOCASE-folded, like the src/dst columns: a re-emit that differs only
|
|
777
|
+
// in case updates the stored row, so it must also keep it (corecorr-05).
|
|
778
|
+
batchEdgeKeys.add(`${org}${RSEP}${nocaseKey(src)}${RSEP}${nocaseKey(String(f.dstQualifiedName))}${RSEP}${f.relType}`);
|
|
779
|
+
let set = srcByOrg.get(org);
|
|
780
|
+
if (!set) {
|
|
781
|
+
set = new Set();
|
|
782
|
+
srcByOrg.set(org, set);
|
|
783
|
+
}
|
|
784
|
+
set.add(src);
|
|
785
|
+
}
|
|
786
|
+
if (extras) {
|
|
787
|
+
const org = String(extras.orgId);
|
|
788
|
+
let set = srcByOrg.get(org);
|
|
789
|
+
if (!set) {
|
|
790
|
+
set = new Set();
|
|
791
|
+
srcByOrg.set(org, set);
|
|
792
|
+
}
|
|
793
|
+
for (const q of extras.qnames)
|
|
794
|
+
set.add(q);
|
|
795
|
+
}
|
|
796
|
+
if (bulk && freshOrg !== null) {
|
|
797
|
+
const fresh = srcByOrg.get(freshOrg);
|
|
798
|
+
if (fresh) {
|
|
799
|
+
// Count per distinct NOCASE source, matching how SQLite would probe.
|
|
800
|
+
const seen = new Set();
|
|
801
|
+
for (const src of [...fresh]) {
|
|
802
|
+
const k = `${freshOrg}${KSEP}${nocaseKey(src)}`;
|
|
803
|
+
if (!bulk.edgeSrcs.has(k)) {
|
|
804
|
+
fresh.delete(src);
|
|
805
|
+
if (!seen.has(k))
|
|
806
|
+
bulk.stats.reconcileSkipped += 1;
|
|
807
|
+
}
|
|
808
|
+
else if (!seen.has(k)) {
|
|
809
|
+
bulk.stats.reconcileProbed += 1;
|
|
810
|
+
}
|
|
811
|
+
seen.add(k);
|
|
812
|
+
}
|
|
813
|
+
if (fresh.size === 0)
|
|
814
|
+
srcByOrg.delete(freshOrg);
|
|
815
|
+
}
|
|
816
|
+
}
|
|
817
|
+
}
|
|
818
|
+
if (bulk && freshOrg !== null) {
|
|
819
|
+
for (const f of facts) {
|
|
820
|
+
if (String(f.orgId) === freshOrg) {
|
|
821
|
+
bulk.edgeSrcs.add(`${freshOrg}${KSEP}${nocaseKey(String(f.srcQualifiedName))}`);
|
|
822
|
+
}
|
|
823
|
+
}
|
|
824
|
+
}
|
|
373
825
|
let inserted = 0;
|
|
374
826
|
let updated = 0;
|
|
375
827
|
let unchanged = 0;
|
|
828
|
+
let deleted = 0;
|
|
376
829
|
const apply = this.db.transaction(() => {
|
|
830
|
+
// Reconcile: read every prior outgoing edge of the owned sources once,
|
|
831
|
+
// across all edge tables (one UNION ALL statement per org), BEFORE this
|
|
832
|
+
// batch writes. It answers the per-fact existence probe for those
|
|
833
|
+
// sources (kept current as the batch inserts / updates, so a repeat
|
|
834
|
+
// within the batch sees the earlier write, as the probe would) and is
|
|
835
|
+
// the prune candidate set below. Rows this batch inserts are always in
|
|
836
|
+
// batchEdgeKeys, so pruning against the pre-write snapshot deletes
|
|
837
|
+
// exactly the rows the old post-write scan deleted.
|
|
838
|
+
const prior = new Map();
|
|
839
|
+
const priorRows = [];
|
|
840
|
+
const covered = new Set();
|
|
841
|
+
const deferredSeen = new Map();
|
|
842
|
+
if (reconcile && srcByOrg.size > 0) {
|
|
843
|
+
for (const [org, srcSet] of srcByOrg) {
|
|
844
|
+
for (const src of srcSet)
|
|
845
|
+
covered.add(`${org}${KSEP}${nocaseKey(src)}`);
|
|
846
|
+
for (const { relType, r } of this.priorOutgoingEdges(org, [...srcSet])) {
|
|
847
|
+
priorRows.push({ org, relType, r });
|
|
848
|
+
prior.set(`${org}${RSEP}${nocaseKey(r.s)}${RSEP}${nocaseKey(r.d)}${RSEP}${relType}`, {
|
|
849
|
+
attributes: r.a,
|
|
850
|
+
id: r.id,
|
|
851
|
+
});
|
|
852
|
+
}
|
|
853
|
+
}
|
|
854
|
+
}
|
|
377
855
|
for (const [relType, bucket] of buckets) {
|
|
378
856
|
const tbl = this.ensureEdgeTable(relType);
|
|
379
|
-
const selectStmt = this.
|
|
380
|
-
const insertStmt = this.
|
|
381
|
-
const updateChangedStmt = this.
|
|
382
|
-
const updateSeenStmt = this.
|
|
857
|
+
const selectStmt = this.stmt(`SELECT attributes, first_seen_at FROM ${tbl} WHERE org_id = ? AND src_qname = ? AND dst_qname = ?`);
|
|
858
|
+
const insertStmt = this.stmt(`INSERT INTO ${tbl}(org_id, src_qname, dst_qname, attributes, first_seen_at, last_seen_at) VALUES (?, ?, ?, ?, ?, ?)`);
|
|
859
|
+
const updateChangedStmt = this.stmt(`UPDATE ${tbl} SET attributes = ?, last_seen_at = ? WHERE org_id = ? AND src_qname = ? AND dst_qname = ?`);
|
|
860
|
+
const updateSeenStmt = this.stmt(`UPDATE ${tbl} SET last_seen_at = ? WHERE org_id = ? AND src_qname = ? AND dst_qname = ?`);
|
|
861
|
+
// M-3: a re-parsed source re-emits its guessed dst (`(?)` arity
|
|
862
|
+
// guess, `ApexClass:` interface name, formula hop guess); the edge a
|
|
863
|
+
// resolver rewrote it to is still present here (pruned below), so the
|
|
864
|
+
// fresh guess inherits that edge's first_seen_at. Probed only for
|
|
865
|
+
// sources this reconcile batch owns and that may have prior rows.
|
|
866
|
+
const rewrittenFromStmt = reconcile
|
|
867
|
+
? this.stmt(`SELECT MIN(first_seen_at) AS t FROM ${tbl} WHERE org_id = ? AND src_qname = ? AND (
|
|
868
|
+
json_extract(attributes, '$.arityResolvedFrom') = ? COLLATE NOCASE OR
|
|
869
|
+
json_extract(attributes, '$.retargetedFrom') = ? COLLATE NOCASE OR
|
|
870
|
+
json_extract(attributes, '$.hopResolvedFrom') = ? COLLATE NOCASE)`)
|
|
871
|
+
: null;
|
|
383
872
|
for (const fact of bucket) {
|
|
384
|
-
const
|
|
873
|
+
const org = String(fact.orgId);
|
|
874
|
+
const src = String(fact.srcQualifiedName);
|
|
875
|
+
const priorKey = covered.has(`${org}${KSEP}${nocaseKey(src)}`)
|
|
876
|
+
? `${org}${RSEP}${nocaseKey(src)}${RSEP}${nocaseKey(String(fact.dstQualifiedName))}${RSEP}${relType}`
|
|
877
|
+
: null;
|
|
878
|
+
const existing = (priorKey !== null
|
|
879
|
+
? prior.get(priorKey)
|
|
880
|
+
: selectStmt.get(fact.orgId, fact.srcQualifiedName, fact.dstQualifiedName));
|
|
385
881
|
const attrsJson = JSON.stringify(fact.attributes);
|
|
882
|
+
if (priorKey !== null && existing?.attributes !== attrsJson) {
|
|
883
|
+
prior.set(priorKey, existing?.id !== undefined
|
|
884
|
+
? { attributes: attrsJson, id: existing.id }
|
|
885
|
+
: { attributes: attrsJson });
|
|
886
|
+
}
|
|
386
887
|
if (!existing) {
|
|
387
|
-
|
|
888
|
+
let firstSeenAt = fact.firstSeenAt;
|
|
889
|
+
if (rewrittenFromStmt &&
|
|
890
|
+
srcByOrg.get(String(fact.orgId))?.has(String(fact.srcQualifiedName))) {
|
|
891
|
+
const dst = String(fact.dstQualifiedName);
|
|
892
|
+
if (dst.endsWith("(?)") ||
|
|
893
|
+
dst.startsWith("ApexClass:") ||
|
|
894
|
+
fact.attributes.hopRelationship != null) {
|
|
895
|
+
const row = rewrittenFromStmt.get(fact.orgId, fact.srcQualifiedName, dst, dst, dst);
|
|
896
|
+
if (row?.t != null && row.t < firstSeenAt)
|
|
897
|
+
firstSeenAt = row.t;
|
|
898
|
+
}
|
|
899
|
+
}
|
|
900
|
+
insertStmt.run(fact.orgId, fact.srcQualifiedName, fact.dstQualifiedName, attrsJson, firstSeenAt, fact.lastSeenAt);
|
|
388
901
|
inserted += 1;
|
|
389
902
|
}
|
|
390
903
|
else if (existing.attributes !== attrsJson) {
|
|
904
|
+
// Runs now, so an earlier deferred last_seen_at write is moot.
|
|
905
|
+
if (existing.id !== undefined)
|
|
906
|
+
deferredSeen.get(relType)?.delete(existing.id);
|
|
391
907
|
updateChangedStmt.run(attrsJson, fact.lastSeenAt, fact.orgId, fact.srcQualifiedName, fact.dstQualifiedName);
|
|
392
908
|
updated += 1;
|
|
393
909
|
}
|
|
910
|
+
else if (existing.id !== undefined) {
|
|
911
|
+
// A prior row re-emitted unchanged: its last_seen_at is written
|
|
912
|
+
// below, in one statement per table (Map order = last write wins).
|
|
913
|
+
let ids = deferredSeen.get(relType);
|
|
914
|
+
if (!ids) {
|
|
915
|
+
ids = new Map();
|
|
916
|
+
deferredSeen.set(relType, ids);
|
|
917
|
+
}
|
|
918
|
+
ids.delete(existing.id);
|
|
919
|
+
ids.set(existing.id, fact.lastSeenAt);
|
|
920
|
+
unchanged += 1;
|
|
921
|
+
}
|
|
394
922
|
else {
|
|
395
923
|
updateSeenStmt.run(fact.lastSeenAt, fact.orgId, fact.srcQualifiedName, fact.dstQualifiedName);
|
|
396
924
|
unchanged += 1;
|
|
397
925
|
}
|
|
398
926
|
}
|
|
399
927
|
}
|
|
928
|
+
// Deferred last_seen_at writes, grouped by value. Nothing in this call
|
|
929
|
+
// reads last_seen_at, and a later changed-attributes update of the same
|
|
930
|
+
// row dropped its entry above, so the rows end exactly as per-fact
|
|
931
|
+
// UPDATEs would leave them.
|
|
932
|
+
for (const [relType, ids] of deferredSeen) {
|
|
933
|
+
const tbl = this.edgeRelCache.get(relType);
|
|
934
|
+
const byValue = new Map();
|
|
935
|
+
for (const [id, seen] of ids) {
|
|
936
|
+
let list = byValue.get(seen);
|
|
937
|
+
if (!list) {
|
|
938
|
+
list = [];
|
|
939
|
+
byValue.set(seen, list);
|
|
940
|
+
}
|
|
941
|
+
list.push(id);
|
|
942
|
+
}
|
|
943
|
+
for (const [seen, list] of byValue) {
|
|
944
|
+
this.stmt(`UPDATE ${tbl} SET last_seen_at = @seen WHERE rowid IN (SELECT value FROM json_each(@ids))`).run({ seen, ids: JSON.stringify(list) });
|
|
945
|
+
}
|
|
946
|
+
}
|
|
947
|
+
// Prune immortal edges: for every source this batch owns, drop any
|
|
948
|
+
// OUTGOING edge it no longer emits. Scoped to (org_id, src_qname) so it's
|
|
949
|
+
// an index-served point lookup on each edge table's PK — NOT a full-table
|
|
950
|
+
// scan (which would be O(members×edges) per ingest and would read other
|
|
951
|
+
// orgs' rows). Edges produced by a different pass — MCD baseline / the
|
|
952
|
+
// reflection walker (attributes.source) and the arity resolver
|
|
953
|
+
// (attributes.resolvedBy) — are NOT touched: the parser batch never
|
|
954
|
+
// contains them, and the pre-fan-out MCD pass never re-adds them, so
|
|
955
|
+
// pruning them would silently destroy long-tail dependency coverage.
|
|
956
|
+
// Inbound edges (dst = source) are left intact regardless.
|
|
957
|
+
// Candidates are the pre-write snapshot read above (see there).
|
|
958
|
+
for (const { org, relType, r } of priorRows) {
|
|
959
|
+
const k = `${org}${RSEP}${nocaseKey(r.s)}${RSEP}${nocaseKey(r.d)}${RSEP}${relType}`;
|
|
960
|
+
if (batchEdgeKeys.has(k))
|
|
961
|
+
continue; // re-emitted this batch — keep
|
|
962
|
+
let attrs = {};
|
|
963
|
+
try {
|
|
964
|
+
attrs = r.a ? JSON.parse(r.a) : {};
|
|
965
|
+
}
|
|
966
|
+
catch {
|
|
967
|
+
attrs = {};
|
|
968
|
+
}
|
|
969
|
+
// Foreign-pass edge (MCD/reflection/arity) — not this parser's to prune.
|
|
970
|
+
if (attrs.source != null || attrs.resolvedBy != null)
|
|
971
|
+
continue;
|
|
972
|
+
const tbl = this.edgeRelCache.get(relType);
|
|
973
|
+
this.stmt(`DELETE FROM ${tbl} WHERE org_id = ? AND src_qname = ? AND dst_qname = ?`).run(org, r.s, r.d);
|
|
974
|
+
deleted += 1;
|
|
975
|
+
}
|
|
400
976
|
});
|
|
401
977
|
apply();
|
|
402
|
-
return { inserted, updated, unchanged };
|
|
978
|
+
return { inserted, updated, unchanged, ...(reconcile ? { deleted } : {}) };
|
|
979
|
+
}
|
|
980
|
+
/**
|
|
981
|
+
* Every outgoing edge of `srcs` (NOCASE, like the src_qname column) in every
|
|
982
|
+
* edge table, in one statement: a UNION ALL of per-table PK range lookups
|
|
983
|
+
* with the sources passed once as a JSON array, materialized as a CTE (no
|
|
984
|
+
* IN-list variable cap, one cached statement whatever the number of
|
|
985
|
+
* sources). Measured ~1.4x faster than one IN-list query per table.
|
|
986
|
+
*/
|
|
987
|
+
priorOutgoingEdges(org, srcs) {
|
|
988
|
+
if (this.edgeRelCache.size === 0 || srcs.length === 0)
|
|
989
|
+
return [];
|
|
990
|
+
let q = this.priorEdgesQuery;
|
|
991
|
+
if (!q || q.map !== this.edgeRelCache || q.size !== this.edgeRelCache.size) {
|
|
992
|
+
const tables = [...this.edgeRelCache];
|
|
993
|
+
const sqls = chunks(tables.map(([, tbl], i) => `SELECT ${i} AS t, rowid AS id, src_qname AS s, dst_qname AS d, attributes AS a FROM ${tbl} WHERE org_id = @org AND src_qname IN q`), MAX_UNION_ARMS).map((arms) => `WITH q(v) AS MATERIALIZED (SELECT value FROM json_each(@srcs)) ${arms.join(" UNION ALL ")}`);
|
|
994
|
+
q = { map: this.edgeRelCache, size: tables.length, rels: tables.map(([r]) => r), sqls };
|
|
995
|
+
this.priorEdgesQuery = q;
|
|
996
|
+
}
|
|
997
|
+
const rels = q.rels;
|
|
998
|
+
const params = { org, srcs: JSON.stringify(srcs) };
|
|
999
|
+
const out = [];
|
|
1000
|
+
for (const sql of q.sqls) {
|
|
1001
|
+
for (const r of this.stmt(sql).all(params)) {
|
|
1002
|
+
out.push({ relType: rels[r.t], r });
|
|
1003
|
+
}
|
|
1004
|
+
}
|
|
1005
|
+
return out;
|
|
403
1006
|
}
|
|
404
1007
|
getNode(orgId, qname) {
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
.get(orgId, qname);
|
|
1008
|
+
this.syncCaches();
|
|
1009
|
+
const idx = this.stmt("SELECT label FROM _sfgraph_node_index WHERE org_id = ? AND qualified_name = ?").get(orgId, qname);
|
|
408
1010
|
if (!idx)
|
|
409
1011
|
return null;
|
|
410
1012
|
const tbl = this.nodeLabelCache.get(idx.label);
|
|
411
1013
|
if (!tbl)
|
|
412
1014
|
return null;
|
|
413
|
-
const row = this.
|
|
414
|
-
.prepare(`SELECT org_id, qualified_name, attributes, source_hash, first_seen_at, last_seen_at, last_modified_at FROM ${tbl} WHERE org_id = ? AND qualified_name = ?`)
|
|
415
|
-
.get(orgId, qname);
|
|
1015
|
+
const row = this.stmt(`SELECT org_id, qualified_name, attributes, source_hash, first_seen_at, last_seen_at, last_modified_at FROM ${tbl} WHERE org_id = ? AND qualified_name = ?`).get(orgId, qname);
|
|
416
1016
|
if (!row)
|
|
417
1017
|
return null;
|
|
418
1018
|
return rowToNode(row, idx.label);
|
|
419
1019
|
}
|
|
420
1020
|
listNodesByLabel(orgId, label, limit) {
|
|
421
1021
|
validateLabel(label);
|
|
1022
|
+
this.syncCaches();
|
|
422
1023
|
const tbl = this.nodeLabelCache.get(label);
|
|
423
1024
|
if (!tbl)
|
|
424
1025
|
return [];
|
|
@@ -426,40 +1027,95 @@ export class SqliteGraphStore {
|
|
|
426
1027
|
const params = [orgId];
|
|
427
1028
|
if (typeof limit === "number")
|
|
428
1029
|
params.push(limit);
|
|
429
|
-
const rows = this.
|
|
1030
|
+
const rows = this.stmt(sql).all(...params);
|
|
430
1031
|
return rows.map((r) => rowToNode(r, label));
|
|
431
1032
|
}
|
|
1033
|
+
/**
|
|
1034
|
+
* listNodesByLabel's qualified names, without the attributes JSON (~6x
|
|
1035
|
+
* cheaper on attribute-heavy labels). `ORDER BY rowid` pins the order
|
|
1036
|
+
* listNodesByLabel gets from its `<tbl>_org` index scan: selecting only
|
|
1037
|
+
* qualified_name would otherwise make the covering PK index the cheapest
|
|
1038
|
+
* plan, which yields NOCASE name order (and a different LIMIT subset).
|
|
1039
|
+
*/
|
|
1040
|
+
listQnamesByLabel(orgId, label, limit) {
|
|
1041
|
+
validateLabel(label);
|
|
1042
|
+
this.syncCaches();
|
|
1043
|
+
const tbl = this.nodeLabelCache.get(label);
|
|
1044
|
+
if (!tbl)
|
|
1045
|
+
return [];
|
|
1046
|
+
const sql = `SELECT qualified_name FROM ${tbl} WHERE org_id = ? ORDER BY rowid${typeof limit === "number" ? " LIMIT ?" : ""}`;
|
|
1047
|
+
const params = [orgId];
|
|
1048
|
+
if (typeof limit === "number")
|
|
1049
|
+
params.push(limit);
|
|
1050
|
+
const rows = this.stmt(sql).all(...params);
|
|
1051
|
+
return rows.map((r) => asQualifiedName(r.qualified_name));
|
|
1052
|
+
}
|
|
1053
|
+
/**
|
|
1054
|
+
* Every edge table's rows with `col = @q`, as ONE statement: a UNION ALL of
|
|
1055
|
+
* the per-table queries listEdgesFrom / listEdgesTo used to run one by one,
|
|
1056
|
+
* in edgeRelCache order. SQLite runs a UNION ALL's arms in order, each with
|
|
1057
|
+
* the plan it gets on its own, so the rows (and their order) are exactly
|
|
1058
|
+
* the concatenation of the per-table results. One statement step per
|
|
1059
|
+
* traversal node instead of one per edge table (~2x faster traces).
|
|
1060
|
+
*/
|
|
1061
|
+
edgesAllTables(col, orgId, q) {
|
|
1062
|
+
let u = this.unionEdgeQueries[col];
|
|
1063
|
+
if (!u || u.map !== this.edgeRelCache || u.size !== this.edgeRelCache.size) {
|
|
1064
|
+
const tables = [...this.edgeRelCache];
|
|
1065
|
+
u = {
|
|
1066
|
+
map: this.edgeRelCache,
|
|
1067
|
+
size: tables.length,
|
|
1068
|
+
rels: tables.map(([r]) => r),
|
|
1069
|
+
// Consecutive groups of tables, run in order (SQLite caps compound arms).
|
|
1070
|
+
sqls: chunks(tables.map(([, tbl], i) => `SELECT ${i} AS t, org_id, src_qname, dst_qname, attributes, first_seen_at, last_seen_at FROM ${tbl} WHERE org_id = @org AND ${col} = @q`), MAX_UNION_ARMS).map((arms) => arms.join(" UNION ALL ")),
|
|
1071
|
+
};
|
|
1072
|
+
this.unionEdgeQueries[col] = u;
|
|
1073
|
+
}
|
|
1074
|
+
const rels = u.rels;
|
|
1075
|
+
const out = [];
|
|
1076
|
+
for (const sql of u.sqls) {
|
|
1077
|
+
const rows = this.stmt(sql).all({ org: orgId, q });
|
|
1078
|
+
for (const r of rows)
|
|
1079
|
+
out.push(rowToEdge(r, rels[r.t]));
|
|
1080
|
+
}
|
|
1081
|
+
return out;
|
|
1082
|
+
}
|
|
432
1083
|
listEdgesFrom(orgId, src, relType) {
|
|
1084
|
+
this.syncCaches();
|
|
1085
|
+
if (!relType)
|
|
1086
|
+
return this.edgesAllTables("src_qname", orgId, src);
|
|
433
1087
|
const out = [];
|
|
434
|
-
const types =
|
|
1088
|
+
const types = [relType];
|
|
435
1089
|
for (const t of types) {
|
|
436
1090
|
const tbl = this.edgeRelCache.get(t);
|
|
437
1091
|
if (!tbl)
|
|
438
1092
|
continue;
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
.all(orgId, src);
|
|
1093
|
+
// perf-05: cached per table; traversals call this once per BFS node.
|
|
1094
|
+
const rows = this.stmt(`SELECT org_id, src_qname, dst_qname, attributes, first_seen_at, last_seen_at FROM ${tbl} WHERE org_id = ? AND src_qname = ?`).all(orgId, src);
|
|
442
1095
|
for (const r of rows)
|
|
443
1096
|
out.push(rowToEdge(r, t));
|
|
444
1097
|
}
|
|
445
1098
|
return out;
|
|
446
1099
|
}
|
|
447
1100
|
listEdgesTo(orgId, dst, relType) {
|
|
1101
|
+
this.syncCaches();
|
|
1102
|
+
if (!relType)
|
|
1103
|
+
return this.edgesAllTables("dst_qname", orgId, dst);
|
|
448
1104
|
const out = [];
|
|
449
|
-
const types =
|
|
1105
|
+
const types = [relType];
|
|
450
1106
|
for (const t of types) {
|
|
451
1107
|
const tbl = this.edgeRelCache.get(t);
|
|
452
1108
|
if (!tbl)
|
|
453
1109
|
continue;
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
.all(orgId, dst);
|
|
1110
|
+
// perf-05: cached per table; traversals call this once per BFS node.
|
|
1111
|
+
const rows = this.stmt(`SELECT org_id, src_qname, dst_qname, attributes, first_seen_at, last_seen_at FROM ${tbl} WHERE org_id = ? AND dst_qname = ?`).all(orgId, dst);
|
|
457
1112
|
for (const r of rows)
|
|
458
1113
|
out.push(rowToEdge(r, t));
|
|
459
1114
|
}
|
|
460
1115
|
return out;
|
|
461
1116
|
}
|
|
462
1117
|
listEdgesByDstLike(orgId, pattern, relType, limit) {
|
|
1118
|
+
this.syncCaches();
|
|
463
1119
|
const out = [];
|
|
464
1120
|
const types = relType ? [relType] : Array.from(this.edgeRelCache.keys());
|
|
465
1121
|
const cap = limit && limit > 0 ? limit : Number.POSITIVE_INFINITY;
|
|
@@ -469,17 +1125,16 @@ export class SqliteGraphStore {
|
|
|
469
1125
|
const tbl = this.edgeRelCache.get(t);
|
|
470
1126
|
if (!tbl)
|
|
471
1127
|
continue;
|
|
1128
|
+
// LIMIT -1 is SQLite's "no limit": one cached statement per table.
|
|
472
1129
|
const remaining = Number.isFinite(cap) ? cap - out.length : -1;
|
|
473
|
-
const
|
|
474
|
-
? `SELECT org_id, src_qname, dst_qname, attributes, first_seen_at, last_seen_at FROM ${tbl} WHERE org_id = ? AND dst_qname LIKE ? LIMIT ${remaining}`
|
|
475
|
-
: `SELECT org_id, src_qname, dst_qname, attributes, first_seen_at, last_seen_at FROM ${tbl} WHERE org_id = ? AND dst_qname LIKE ?`;
|
|
476
|
-
const rows = this.db.prepare(sql).all(orgId, pattern);
|
|
1130
|
+
const rows = this.stmt(`SELECT org_id, src_qname, dst_qname, attributes, first_seen_at, last_seen_at FROM ${tbl} WHERE org_id = ? AND dst_qname LIKE ? LIMIT ?`).all(orgId, pattern, remaining);
|
|
477
1131
|
for (const r of rows)
|
|
478
1132
|
out.push(rowToEdge(r, t));
|
|
479
1133
|
}
|
|
480
1134
|
return out;
|
|
481
1135
|
}
|
|
482
1136
|
deleteEdge(orgId, src, dst, relType) {
|
|
1137
|
+
this.syncCaches();
|
|
483
1138
|
const tbl = this.edgeRelCache.get(relType);
|
|
484
1139
|
if (!tbl)
|
|
485
1140
|
return;
|
|
@@ -495,6 +1150,7 @@ export class SqliteGraphStore {
|
|
|
495
1150
|
* field refs whose CustomObject wasn't ingested).
|
|
496
1151
|
*/
|
|
497
1152
|
listDanglingEdges(orgId, limit) {
|
|
1153
|
+
this.syncCaches();
|
|
498
1154
|
const out = [];
|
|
499
1155
|
const cap = limit && limit > 0 ? limit : Number.POSITIVE_INFINITY;
|
|
500
1156
|
for (const [relType, tbl] of this.edgeRelCache) {
|
|
@@ -515,12 +1171,41 @@ export class SqliteGraphStore {
|
|
|
515
1171
|
}
|
|
516
1172
|
return out;
|
|
517
1173
|
}
|
|
1174
|
+
/**
|
|
1175
|
+
* Streaming form of `listDanglingEdges`: same SQL, tables and order, one
|
|
1176
|
+
* row at a time and without parsing attributes, so an org with ~1M
|
|
1177
|
+
* dangling edges never holds them all at once.
|
|
1178
|
+
*/
|
|
1179
|
+
*iterateDanglingEdges(orgId, limit) {
|
|
1180
|
+
this.syncCaches();
|
|
1181
|
+
const cap = limit && limit > 0 ? limit : Number.POSITIVE_INFINITY;
|
|
1182
|
+
let seen = 0;
|
|
1183
|
+
for (const [relType, tbl] of [...this.edgeRelCache]) {
|
|
1184
|
+
if (seen >= cap)
|
|
1185
|
+
return;
|
|
1186
|
+
const remaining = Number.isFinite(cap) ? cap - seen : -1;
|
|
1187
|
+
const limitClause = remaining > 0 ? `LIMIT ${remaining}` : "";
|
|
1188
|
+
const rows = this.db
|
|
1189
|
+
.prepare(`SELECT e.src_qname, e.dst_qname, e.attributes
|
|
1190
|
+
FROM ${tbl} e
|
|
1191
|
+
LEFT JOIN _sfgraph_node_index n
|
|
1192
|
+
ON n.org_id = e.org_id AND n.qualified_name = e.dst_qname
|
|
1193
|
+
WHERE e.org_id = ? AND n.qualified_name IS NULL
|
|
1194
|
+
${limitClause}`)
|
|
1195
|
+
.iterate(orgId);
|
|
1196
|
+
for (const r of rows) {
|
|
1197
|
+
seen++;
|
|
1198
|
+
yield { relType, src: r.src_qname, dst: r.dst_qname, attributes: r.attributes };
|
|
1199
|
+
}
|
|
1200
|
+
}
|
|
1201
|
+
}
|
|
518
1202
|
/**
|
|
519
1203
|
* Return every qualified name persisted for `orgId` across all label tables.
|
|
520
1204
|
* Used by full-sync deletion detection to compute the difference between
|
|
521
1205
|
* what was previously known and what the current sync touched.
|
|
522
1206
|
*/
|
|
523
1207
|
listAllQnames(orgId) {
|
|
1208
|
+
this.syncCaches();
|
|
524
1209
|
const out = [];
|
|
525
1210
|
for (const tbl of this.nodeLabelCache.values()) {
|
|
526
1211
|
const rows = this.db
|
|
@@ -537,9 +1222,11 @@ export class SqliteGraphStore {
|
|
|
537
1222
|
* exposing the private nodeLabelCache. Order is undefined.
|
|
538
1223
|
*/
|
|
539
1224
|
listAllLabels() {
|
|
1225
|
+
this.syncCaches();
|
|
540
1226
|
return [...this.nodeLabelCache.keys()];
|
|
541
1227
|
}
|
|
542
1228
|
countNodes(orgId) {
|
|
1229
|
+
this.syncCaches();
|
|
543
1230
|
let total = 0;
|
|
544
1231
|
for (const tbl of this.nodeLabelCache.values()) {
|
|
545
1232
|
const row = this.db.prepare(`SELECT COUNT(*) AS c FROM ${tbl} WHERE org_id = ?`).get(orgId);
|
|
@@ -557,6 +1244,7 @@ export class SqliteGraphStore {
|
|
|
557
1244
|
* helper is added rather than overloading countNodes.
|
|
558
1245
|
*/
|
|
559
1246
|
countNodesByLabel(orgId, label) {
|
|
1247
|
+
this.syncCaches();
|
|
560
1248
|
const tbl = this.nodeLabelCache.get(label);
|
|
561
1249
|
if (!tbl)
|
|
562
1250
|
return 0;
|
|
@@ -564,6 +1252,7 @@ export class SqliteGraphStore {
|
|
|
564
1252
|
return row?.c ?? 0;
|
|
565
1253
|
}
|
|
566
1254
|
countEdges(orgId) {
|
|
1255
|
+
this.syncCaches();
|
|
567
1256
|
let total = 0;
|
|
568
1257
|
for (const tbl of this.edgeRelCache.values()) {
|
|
569
1258
|
const row = this.db.prepare(`SELECT COUNT(*) AS c FROM ${tbl} WHERE org_id = ?`).get(orgId);
|
|
@@ -575,26 +1264,20 @@ export class SqliteGraphStore {
|
|
|
575
1264
|
return this.db.transaction(fn)();
|
|
576
1265
|
}
|
|
577
1266
|
upsertSnippet(rec) {
|
|
578
|
-
const existing = this.
|
|
579
|
-
.prepare("SELECT source_hash FROM _sfgraph_snippets WHERE org_id = ? AND qualified_name = ?")
|
|
580
|
-
.get(rec.orgId, rec.qualifiedName);
|
|
1267
|
+
const existing = this.stmt("SELECT source_hash FROM _sfgraph_snippets WHERE org_id = ? AND qualified_name = ?").get(rec.orgId, rec.qualifiedName);
|
|
581
1268
|
if (existing && existing.source_hash === rec.sourceHash) {
|
|
582
1269
|
return { inserted: false, updated: false, unchanged: true };
|
|
583
1270
|
}
|
|
584
1271
|
if (existing) {
|
|
585
|
-
this.
|
|
586
|
-
.prepare(`UPDATE _sfgraph_snippets
|
|
1272
|
+
this.stmt(`UPDATE _sfgraph_snippets
|
|
587
1273
|
SET source_format = ?, source_text = ?, start_line = ?, end_line = ?,
|
|
588
1274
|
source_hash = ?, llm_explanation = NULL, explained_at = NULL
|
|
589
|
-
WHERE org_id = ? AND qualified_name = ?`)
|
|
590
|
-
.run(rec.sourceFormat, rec.sourceText, rec.startLine ?? null, rec.endLine ?? null, rec.sourceHash, rec.orgId, rec.qualifiedName);
|
|
1275
|
+
WHERE org_id = ? AND qualified_name = ?`).run(rec.sourceFormat, rec.sourceText, rec.startLine ?? null, rec.endLine ?? null, rec.sourceHash, rec.orgId, rec.qualifiedName);
|
|
591
1276
|
return { inserted: false, updated: true, unchanged: false };
|
|
592
1277
|
}
|
|
593
|
-
this.
|
|
594
|
-
.prepare(`INSERT INTO _sfgraph_snippets
|
|
1278
|
+
this.stmt(`INSERT INTO _sfgraph_snippets
|
|
595
1279
|
(org_id, qualified_name, source_format, source_text, start_line, end_line, source_hash, llm_explanation, explained_at)
|
|
596
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
597
|
-
.run(rec.orgId, rec.qualifiedName, rec.sourceFormat, rec.sourceText, rec.startLine ?? null, rec.endLine ?? null, rec.sourceHash, rec.llmExplanation ?? null, rec.explainedAt ?? null);
|
|
1280
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(rec.orgId, rec.qualifiedName, rec.sourceFormat, rec.sourceText, rec.startLine ?? null, rec.endLine ?? null, rec.sourceHash, rec.llmExplanation ?? null, rec.explainedAt ?? null);
|
|
598
1281
|
return { inserted: true, updated: false, unchanged: false };
|
|
599
1282
|
}
|
|
600
1283
|
getSnippet(orgId, qname) {
|
|
@@ -654,15 +1337,941 @@ export class SqliteGraphStore {
|
|
|
654
1337
|
return rec;
|
|
655
1338
|
});
|
|
656
1339
|
}
|
|
1340
|
+
/**
|
|
1341
|
+
* Replace the FTS doc for (org, qname). The old row is found through the
|
|
1342
|
+
* indexed _sfgraph_node_fts_rowid map and deleted by rowid; a WHERE on the
|
|
1343
|
+
* FTS5 UNINDEXED key columns would scan the whole table (O(n²) per ingest).
|
|
1344
|
+
* Keys compare BINARY, exactly like the old column equality, and the insert
|
|
1345
|
+
* sequence (hence rowids and bm25 tie order) is unchanged.
|
|
1346
|
+
*/
|
|
1347
|
+
upsertNodeFts(orgId, qname, label, body) {
|
|
1348
|
+
try {
|
|
1349
|
+
if (!this.ftsUpsertTx) {
|
|
1350
|
+
this.ftsUpsertTx = this.db.transaction((org, q, lbl, text) => {
|
|
1351
|
+
const bulk = this.bulk;
|
|
1352
|
+
const key = `${org}${KSEP}${q}`;
|
|
1353
|
+
if (bulk && bulk.freshOrg === org && !bulk.ftsKeys.has(key)) {
|
|
1354
|
+
// Fresh org, first write of this key in the window: nothing to delete.
|
|
1355
|
+
bulk.stats.ftsDeleteSkipped += 1;
|
|
1356
|
+
}
|
|
1357
|
+
else {
|
|
1358
|
+
this.deleteNodeFts(org, q);
|
|
1359
|
+
}
|
|
1360
|
+
const info = this.stmt("INSERT INTO _sfgraph_node_fts(qualified_name, org_id, label, body) VALUES (?, ?, ?, ?)").run(q, org, lbl, text);
|
|
1361
|
+
this.stmt(`INSERT INTO _sfgraph_node_fts_rowid(org_id, qualified_name, fts_rowid) VALUES (?, ?, ?)
|
|
1362
|
+
ON CONFLICT(org_id, qualified_name) DO UPDATE SET fts_rowid = excluded.fts_rowid`).run(org, q, info.lastInsertRowid);
|
|
1363
|
+
if (bulk && bulk.freshOrg === org)
|
|
1364
|
+
bulk.ftsKeys.add(key);
|
|
1365
|
+
});
|
|
1366
|
+
}
|
|
1367
|
+
this.ftsUpsertTx(String(orgId), String(qname), label, body);
|
|
1368
|
+
}
|
|
1369
|
+
catch {
|
|
1370
|
+
/* FTS5 unavailable — keyword leg degrades to empty. */
|
|
1371
|
+
}
|
|
1372
|
+
}
|
|
1373
|
+
/** Delete (org, qname)'s FTS doc by rowid via the side table. Best-effort when FTS5 is absent. */
|
|
1374
|
+
deleteNodeFts(orgId, qname) {
|
|
1375
|
+
let row;
|
|
1376
|
+
try {
|
|
1377
|
+
row = this.stmt("SELECT fts_rowid AS r FROM _sfgraph_node_fts_rowid WHERE org_id = ? AND qualified_name = ?").get(orgId, qname);
|
|
1378
|
+
}
|
|
1379
|
+
catch {
|
|
1380
|
+
return; // pre-migration / FTS-less store
|
|
1381
|
+
}
|
|
1382
|
+
if (!row)
|
|
1383
|
+
return;
|
|
1384
|
+
try {
|
|
1385
|
+
this.stmt("DELETE FROM _sfgraph_node_fts WHERE rowid = ?").run(row.r);
|
|
1386
|
+
}
|
|
1387
|
+
catch {
|
|
1388
|
+
/* FTS5 unavailable */
|
|
1389
|
+
}
|
|
1390
|
+
this.stmt("DELETE FROM _sfgraph_node_fts_rowid WHERE org_id = ? AND qualified_name = ?").run(orgId, qname);
|
|
1391
|
+
}
|
|
1392
|
+
// ─── Task 10: bulk-load window for rebuilds ───────────────────────────────
|
|
1393
|
+
/**
|
|
1394
|
+
* Self-heal after a process died inside a bulk window (crash, SIGKILL, OOM,
|
|
1395
|
+
* native checkpoint abort): the DROP INDEXes were durable but endBulk never
|
|
1396
|
+
* rebuilt them. Recreates every missing canonical `_org` / `_rev` index and
|
|
1397
|
+
* clears the marker. Skipped unless the marker is ours or its owner is
|
|
1398
|
+
* stale (same host, dead pid): another live process's window, or any
|
|
1399
|
+
* window opened on another host, is legitimately open. Cheap when nothing is missing:
|
|
1400
|
+
* one sqlite_master read. Best-effort — a busy DB must not fail init().
|
|
1401
|
+
*/
|
|
1402
|
+
/** Task 11c: markers written before the host column get it added (null = this host). */
|
|
1403
|
+
ensureBulkMarkerHost() {
|
|
1404
|
+
const cols = this.db.prepare("PRAGMA table_info(_sfgraph_bulk_window)").all();
|
|
1405
|
+
if (!cols.some((c) => c.name === "host")) {
|
|
1406
|
+
try {
|
|
1407
|
+
this.db.exec("ALTER TABLE _sfgraph_bulk_window ADD COLUMN host TEXT");
|
|
1408
|
+
}
|
|
1409
|
+
catch {
|
|
1410
|
+
/* a concurrent opener added it */
|
|
1411
|
+
}
|
|
1412
|
+
}
|
|
1413
|
+
}
|
|
1414
|
+
healBulkWindow() {
|
|
1415
|
+
try {
|
|
1416
|
+
const hasMarkerTable = this.db
|
|
1417
|
+
.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = '_sfgraph_bulk_window'")
|
|
1418
|
+
.get() !== undefined;
|
|
1419
|
+
if (hasMarkerTable)
|
|
1420
|
+
this.ensureBulkMarkerHost();
|
|
1421
|
+
const marker = hasMarkerTable
|
|
1422
|
+
? this.db
|
|
1423
|
+
.prepare("SELECT pid, host, started_at FROM _sfgraph_bulk_window WHERE id = 1")
|
|
1424
|
+
.get()
|
|
1425
|
+
: undefined;
|
|
1426
|
+
// Heal only our own window or one whose owner is provably gone: same
|
|
1427
|
+
// host, dead pid. A window opened on another host is never touched.
|
|
1428
|
+
if (marker) {
|
|
1429
|
+
const ours = marker.pid === process.pid && isThisHost(marker.host);
|
|
1430
|
+
if (!ours && !ownerIsStale({ ...marker, takenAtMs: marker.started_at }))
|
|
1431
|
+
return;
|
|
1432
|
+
}
|
|
1433
|
+
const objects = this.db
|
|
1434
|
+
.prepare("SELECT type, name FROM sqlite_master WHERE type IN ('table', 'index')")
|
|
1435
|
+
.all();
|
|
1436
|
+
const tables = new Set(objects.filter((o) => o.type === "table").map((o) => o.name));
|
|
1437
|
+
const idx = new Set(objects.filter((o) => o.type === "index").map((o) => o.name));
|
|
1438
|
+
const missing = [];
|
|
1439
|
+
for (const tbl of this.nodeLabelCache.values())
|
|
1440
|
+
if (tables.has(tbl) && !idx.has(`${tbl}_org`))
|
|
1441
|
+
missing.push(nodeIndexDdl(tbl));
|
|
1442
|
+
for (const tbl of this.edgeRelCache.values())
|
|
1443
|
+
if (tables.has(tbl) && !idx.has(`${tbl}_rev`))
|
|
1444
|
+
missing.push(edgeIndexDdl(tbl));
|
|
1445
|
+
if (missing.length === 0 && !marker)
|
|
1446
|
+
return;
|
|
1447
|
+
this.db.transaction(() => {
|
|
1448
|
+
for (const sql of missing)
|
|
1449
|
+
this.db.exec(sql);
|
|
1450
|
+
if (marker)
|
|
1451
|
+
this.db.prepare("DELETE FROM _sfgraph_bulk_window WHERE id = 1").run();
|
|
1452
|
+
})();
|
|
1453
|
+
}
|
|
1454
|
+
catch {
|
|
1455
|
+
/* best-effort: e.g. SQLITE_BUSY; retried on the next init() */
|
|
1456
|
+
}
|
|
1457
|
+
}
|
|
1458
|
+
// ─── Task 11a: embedding status ───────────────────────────────────────────
|
|
1459
|
+
/** Mark the org's vectors as still being written (or done). Best-effort on
|
|
1460
|
+
* a pre-v12 store. */
|
|
1461
|
+
setEmbeddingsPending(orgId, pending) {
|
|
1462
|
+
try {
|
|
1463
|
+
this.stmt("UPDATE _sfgraph_orgs SET embeddings_pending = ? WHERE id = ?").run(pending ? 1 : 0, orgId);
|
|
1464
|
+
}
|
|
1465
|
+
catch {
|
|
1466
|
+
/* pre-v12 store */
|
|
1467
|
+
}
|
|
1468
|
+
}
|
|
1469
|
+
/**
|
|
1470
|
+
* Whether the org's vectors are still being written, and how many texts it
|
|
1471
|
+
* embeds in total (its FTS docs: every embedded text is also an FTS doc).
|
|
1472
|
+
* Null for an unknown org or a store without the v12 column.
|
|
1473
|
+
*/
|
|
1474
|
+
getEmbeddingStatus(orgId) {
|
|
1475
|
+
let row;
|
|
1476
|
+
try {
|
|
1477
|
+
row = this.stmt("SELECT embeddings_pending AS p FROM _sfgraph_orgs WHERE id = ?").get(orgId);
|
|
1478
|
+
}
|
|
1479
|
+
catch {
|
|
1480
|
+
return null;
|
|
1481
|
+
}
|
|
1482
|
+
if (!row)
|
|
1483
|
+
return null;
|
|
1484
|
+
let total = 0;
|
|
1485
|
+
try {
|
|
1486
|
+
total = this.stmt("SELECT COUNT(*) AS c FROM _sfgraph_node_fts_rowid WHERE org_id = ?").get(orgId).c;
|
|
1487
|
+
}
|
|
1488
|
+
catch {
|
|
1489
|
+
/* FTS-less store */
|
|
1490
|
+
}
|
|
1491
|
+
return { pending: row.p === 1, total };
|
|
1492
|
+
}
|
|
1493
|
+
/** Every FTS doc of the org (the exact embed text per qname), for the
|
|
1494
|
+
* embedding backfill. Empty when FTS5 is unavailable. Streams in keyset
|
|
1495
|
+
* pages of `pageSize` rows, so at most one page of bodies is held at once.
|
|
1496
|
+
* Not `.iterate()`: the caller writes vectors between rows (backpressure
|
|
1497
|
+
* can force a dispatch), and a better-sqlite3 connection with an open
|
|
1498
|
+
* iterator rejects writes ("connection is busy"). */
|
|
1499
|
+
*listFtsDocs(orgId, pageSize = 1000) {
|
|
1500
|
+
// Keyset on the (org_id, qualified_name) primary key: each page is an
|
|
1501
|
+
// index range scan, not a re-sort of the whole org.
|
|
1502
|
+
let after = "";
|
|
1503
|
+
for (;;) {
|
|
1504
|
+
let rows;
|
|
1505
|
+
try {
|
|
1506
|
+
rows = this.stmt(`SELECT r.qualified_name AS qname, f.label AS label, f.body AS body
|
|
1507
|
+
FROM _sfgraph_node_fts_rowid r
|
|
1508
|
+
JOIN _sfgraph_node_fts f ON f.rowid = r.fts_rowid
|
|
1509
|
+
WHERE r.org_id = ? AND r.qualified_name > ?
|
|
1510
|
+
ORDER BY r.qualified_name
|
|
1511
|
+
LIMIT ?`).all(orgId, after, pageSize);
|
|
1512
|
+
}
|
|
1513
|
+
catch {
|
|
1514
|
+
return;
|
|
1515
|
+
}
|
|
1516
|
+
yield* rows;
|
|
1517
|
+
if (rows.length < pageSize)
|
|
1518
|
+
return;
|
|
1519
|
+
after = rows[rows.length - 1].qname;
|
|
1520
|
+
}
|
|
1521
|
+
}
|
|
1522
|
+
/** True when `orgId` has no node, edge or FTS rows. Index probes only (LIMIT 1). */
|
|
1523
|
+
isOrgEmpty(orgId) {
|
|
1524
|
+
this.syncCaches();
|
|
1525
|
+
const probe = (tbl) => this.db.prepare(`SELECT 1 FROM ${tbl} WHERE org_id = ? LIMIT 1`).get(orgId) !== undefined;
|
|
1526
|
+
for (const tbl of this.nodeLabelCache.values())
|
|
1527
|
+
if (probe(tbl))
|
|
1528
|
+
return false;
|
|
1529
|
+
for (const tbl of this.edgeRelCache.values())
|
|
1530
|
+
if (probe(tbl))
|
|
1531
|
+
return false;
|
|
1532
|
+
if (probe("_sfgraph_node_index"))
|
|
1533
|
+
return false;
|
|
1534
|
+
try {
|
|
1535
|
+
if (probe("_sfgraph_node_fts_rowid"))
|
|
1536
|
+
return false;
|
|
1537
|
+
}
|
|
1538
|
+
catch {
|
|
1539
|
+
/* table absent */
|
|
1540
|
+
}
|
|
1541
|
+
return true;
|
|
1542
|
+
}
|
|
1543
|
+
/**
|
|
1544
|
+
* Open a bulk-load window for a rebuild: drop the graph tables' secondary
|
|
1545
|
+
* indexes (rebuilt in one pass at endBulk), set journal_mode=WAL,
|
|
1546
|
+
* synchronous=OFF, temp_store=MEMORY, and hold one write transaction per
|
|
1547
|
+
* BULK_TXN_MEMBERS members (advanced by bulkMemberDone). Not re-entrant.
|
|
1548
|
+
*/
|
|
1549
|
+
beginBulk(opts = {}) {
|
|
1550
|
+
if (this.bulk)
|
|
1551
|
+
throw new StorageError("beginBulk: a bulk window is already open");
|
|
1552
|
+
if (this.batch)
|
|
1553
|
+
throw new StorageError("beginBulk: a batch window is open");
|
|
1554
|
+
if (this.db.inTransaction)
|
|
1555
|
+
throw new StorageError("beginBulk: called inside a transaction");
|
|
1556
|
+
this.syncCaches();
|
|
1557
|
+
const fresh = opts.freshOrgId !== undefined && this.isOrgEmpty(opts.freshOrgId)
|
|
1558
|
+
? String(opts.freshOrgId)
|
|
1559
|
+
: null;
|
|
1560
|
+
const batchTxn = opts.batchTransactions !== false;
|
|
1561
|
+
const graphTables = new Set([...this.nodeLabelCache.values(), ...this.edgeRelCache.values()]);
|
|
1562
|
+
// The _sfg_* tables are shared by every org in the file. A fresh org next
|
|
1563
|
+
// to other orgs' rows keeps the existing indexes (their readers, and an
|
|
1564
|
+
// endBulk re-index of every org's rows, would otherwise pay); only tables
|
|
1565
|
+
// created inside the window defer their indexes.
|
|
1566
|
+
const otherOrgsHaveRows = fresh !== null &&
|
|
1567
|
+
[...graphTables].some((t) => this.db.prepare(`SELECT 1 FROM ${t} LIMIT 1`).get() !== undefined);
|
|
1568
|
+
const existing = otherOrgsHaveRows
|
|
1569
|
+
? []
|
|
1570
|
+
: this.db
|
|
1571
|
+
.prepare("SELECT name, tbl_name AS tbl, sql FROM sqlite_master WHERE type = 'index' AND sql IS NOT NULL")
|
|
1572
|
+
.all().filter((i) => graphTables.has(i.tbl));
|
|
1573
|
+
// Durable marker first (autocommit), so a crash after any DROP is healed.
|
|
1574
|
+
this.db.exec(BULK_MARKER_DDL);
|
|
1575
|
+
this.ensureBulkMarkerHost();
|
|
1576
|
+
this.db
|
|
1577
|
+
.prepare("INSERT OR REPLACE INTO _sfgraph_bulk_window(id, pid, started_at, host) VALUES (1, ?, ?, ?)")
|
|
1578
|
+
.run(process.pid, Date.now(), thisHost());
|
|
1579
|
+
// Register the window before touching anything, so a failure part-way
|
|
1580
|
+
// through is undone by endBulk (indexes rebuilt, synchronous restored).
|
|
1581
|
+
this.bulk = {
|
|
1582
|
+
freshOrg: fresh,
|
|
1583
|
+
batchTxn,
|
|
1584
|
+
members: 0,
|
|
1585
|
+
edgeSrcs: new Set(),
|
|
1586
|
+
ftsKeys: new Set(),
|
|
1587
|
+
deferredIndexes: existing.map((i) => ({ name: i.name, table: i.tbl, sql: i.sql })),
|
|
1588
|
+
stats: {
|
|
1589
|
+
active: true,
|
|
1590
|
+
fresh: fresh !== null,
|
|
1591
|
+
members: 0,
|
|
1592
|
+
reconcileSkipped: 0,
|
|
1593
|
+
reconcileProbed: 0,
|
|
1594
|
+
ftsDeleteSkipped: 0,
|
|
1595
|
+
},
|
|
1596
|
+
};
|
|
1597
|
+
try {
|
|
1598
|
+
this.db.pragma("journal_mode = WAL");
|
|
1599
|
+
this.db.pragma("synchronous = OFF");
|
|
1600
|
+
this.db.pragma("temp_store = MEMORY");
|
|
1601
|
+
for (const i of existing)
|
|
1602
|
+
this.db.exec(`DROP INDEX IF EXISTS "${i.name.replace(/"/g, '""')}"`);
|
|
1603
|
+
if (batchTxn)
|
|
1604
|
+
this.db.exec("BEGIN");
|
|
1605
|
+
}
|
|
1606
|
+
catch (e) {
|
|
1607
|
+
try {
|
|
1608
|
+
this.endBulk();
|
|
1609
|
+
}
|
|
1610
|
+
catch {
|
|
1611
|
+
/* the original error matters more; init() heals on next open */
|
|
1612
|
+
}
|
|
1613
|
+
throw e;
|
|
1614
|
+
}
|
|
1615
|
+
}
|
|
1616
|
+
/** Mark one member written; commits and reopens the bulk transaction every BULK_TXN_MEMBERS. */
|
|
1617
|
+
bulkMemberDone() {
|
|
1618
|
+
const bulk = this.bulk;
|
|
1619
|
+
if (!bulk)
|
|
1620
|
+
return;
|
|
1621
|
+
if (bulk.batchTxn && !this.db.inTransaction) {
|
|
1622
|
+
// SQLite rolls back the whole outer BEGIN on SQLITE_FULL / IOERR / NOMEM
|
|
1623
|
+
// inside a member's savepoint: every member since the last commit is lost.
|
|
1624
|
+
throw new StorageError("bulk transaction was rolled back by SQLite; members written since the last commit were lost");
|
|
1625
|
+
}
|
|
1626
|
+
bulk.members += 1;
|
|
1627
|
+
bulk.stats.members = bulk.members;
|
|
1628
|
+
if (bulk.batchTxn && bulk.members % BULK_TXN_MEMBERS === 0 && this.db.inTransaction) {
|
|
1629
|
+
this.db.exec("COMMIT");
|
|
1630
|
+
this.db.exec("BEGIN");
|
|
1631
|
+
}
|
|
1632
|
+
}
|
|
1633
|
+
/**
|
|
1634
|
+
* Close the bulk window: commit, recreate the deferred indexes (with
|
|
1635
|
+
* temp_store=FILE, then the previous temp_store is restored), restore
|
|
1636
|
+
* synchronous=NORMAL. No-op when closed.
|
|
1637
|
+
*/
|
|
1638
|
+
endBulk() {
|
|
1639
|
+
const bulk = this.bulk;
|
|
1640
|
+
if (!bulk)
|
|
1641
|
+
return;
|
|
1642
|
+
this.bulk = null;
|
|
1643
|
+
try {
|
|
1644
|
+
if (bulk.batchTxn && this.db.inTransaction)
|
|
1645
|
+
this.db.exec("COMMIT");
|
|
1646
|
+
}
|
|
1647
|
+
finally {
|
|
1648
|
+
const build = this.db.transaction(() => {
|
|
1649
|
+
for (const i of bulk.deferredIndexes) {
|
|
1650
|
+
const hasTable = this.db
|
|
1651
|
+
.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?")
|
|
1652
|
+
.get(i.table);
|
|
1653
|
+
const hasIndex = this.db
|
|
1654
|
+
.prepare("SELECT 1 FROM sqlite_master WHERE type = 'index' AND name = ?")
|
|
1655
|
+
.get(i.name);
|
|
1656
|
+
if (hasTable && !hasIndex)
|
|
1657
|
+
this.db.exec(i.sql);
|
|
1658
|
+
}
|
|
1659
|
+
});
|
|
1660
|
+
// The index sorter spills past cache_size into temp files; with
|
|
1661
|
+
// temp_store=MEMORY those live in RAM (+~1 GB RSS on a 2.6M-edge org,
|
|
1662
|
+
// never returned by malloc). Spill to disk for the rebuild only.
|
|
1663
|
+
// SQLite refuses to change temp_store inside a transaction.
|
|
1664
|
+
const prevTempStore = this.db.inTransaction
|
|
1665
|
+
? null
|
|
1666
|
+
: Number(this.db.pragma("temp_store", { simple: true }));
|
|
1667
|
+
if (prevTempStore === null)
|
|
1668
|
+
console.warn("sfgraph: endBulk is inside a transaction (the bulk commit failed?); rebuilding indexes with the connection's temp_store (MEMORY), which can use much more RAM");
|
|
1669
|
+
try {
|
|
1670
|
+
if (prevTempStore !== null && prevTempStore !== 1)
|
|
1671
|
+
this.db.pragma("temp_store = FILE");
|
|
1672
|
+
build();
|
|
1673
|
+
this.db.prepare("DELETE FROM _sfgraph_bulk_window WHERE id = 1").run();
|
|
1674
|
+
}
|
|
1675
|
+
finally {
|
|
1676
|
+
try {
|
|
1677
|
+
if (prevTempStore !== null && prevTempStore !== 1 && !this.db.inTransaction)
|
|
1678
|
+
this.db.pragma(`temp_store = ${prevTempStore}`);
|
|
1679
|
+
}
|
|
1680
|
+
finally {
|
|
1681
|
+
this.db.pragma("synchronous = NORMAL");
|
|
1682
|
+
}
|
|
1683
|
+
}
|
|
1684
|
+
}
|
|
1685
|
+
}
|
|
1686
|
+
/** Snapshot of the current bulk window's counters (inactive zeros when closed). */
|
|
1687
|
+
bulkStats() {
|
|
1688
|
+
if (this.bulk)
|
|
1689
|
+
return { ...this.bulk.stats };
|
|
1690
|
+
return {
|
|
1691
|
+
active: false,
|
|
1692
|
+
fresh: false,
|
|
1693
|
+
members: 0,
|
|
1694
|
+
reconcileSkipped: 0,
|
|
1695
|
+
reconcileProbed: 0,
|
|
1696
|
+
ftsDeleteSkipped: 0,
|
|
1697
|
+
};
|
|
1698
|
+
}
|
|
1699
|
+
// ─── Live-graph write batches (non-rebuild ingests) ────────────────────────
|
|
1700
|
+
/**
|
|
1701
|
+
* Group a non-rebuild ingest's per-member writes into one transaction per
|
|
1702
|
+
* BATCH_TXN_MEMBERS members or BATCH_TXN_MAX_MS, whichever comes first.
|
|
1703
|
+
* Unlike beginBulk nothing about durability changes: indexes stay, and
|
|
1704
|
+
* synchronous / journal_mode keep their values, so a crash loses at most the
|
|
1705
|
+
* open (uncommitted) batch and leaves the last committed state intact.
|
|
1706
|
+
*
|
|
1707
|
+
* Each transaction starts lazily at batchMemberStart with BEGIN IMMEDIATE (the
|
|
1708
|
+
* write lock is taken up front, waiting up to BUSY_TIMEOUT_MS; a deferred
|
|
1709
|
+
* BEGIN could fail every later member with SQLITE_BUSY_SNAPSHOT after another
|
|
1710
|
+
* connection's commit) and is committed only between members, so a
|
|
1711
|
+
* concurrent reader sees each member whole or not at all. An idle timer
|
|
1712
|
+
* commits a transaction left open while the member stream is slow (a live
|
|
1713
|
+
* org's retrieve), so the lock is never held much past maxMs. The writes are
|
|
1714
|
+
* the same statements as without a batch, so the graph is identical.
|
|
1715
|
+
*/
|
|
1716
|
+
beginBatch(opts = {}) {
|
|
1717
|
+
if (this.bulk)
|
|
1718
|
+
throw new StorageError("beginBatch: a bulk window is open");
|
|
1719
|
+
if (this.batch)
|
|
1720
|
+
throw new StorageError("beginBatch: a batch window is already open");
|
|
1721
|
+
if (this.db.inTransaction)
|
|
1722
|
+
throw new StorageError("beginBatch: called inside a transaction");
|
|
1723
|
+
this.batch = {
|
|
1724
|
+
maxMembers: Math.max(1, opts.maxMembers ?? BATCH_TXN_MEMBERS),
|
|
1725
|
+
maxMs: Math.max(1, opts.maxMs ?? BATCH_TXN_MAX_MS),
|
|
1726
|
+
open: false,
|
|
1727
|
+
openedAt: 0,
|
|
1728
|
+
inTxn: 0,
|
|
1729
|
+
timer: null,
|
|
1730
|
+
stats: { active: true, members: 0, commits: 0, beginFailures: 0 },
|
|
1731
|
+
};
|
|
1732
|
+
}
|
|
1733
|
+
/** Before a member's writes: open the batch transaction if none is open. */
|
|
1734
|
+
batchMemberStart() {
|
|
1735
|
+
const batch = this.batch;
|
|
1736
|
+
if (!batch || batch.open || this.db.inTransaction)
|
|
1737
|
+
return;
|
|
1738
|
+
try {
|
|
1739
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
1740
|
+
}
|
|
1741
|
+
catch {
|
|
1742
|
+
// Another writer kept the lock past busy_timeout: this member is
|
|
1743
|
+
// written per statement, exactly as without a batch.
|
|
1744
|
+
batch.stats.beginFailures += 1;
|
|
1745
|
+
return;
|
|
1746
|
+
}
|
|
1747
|
+
batch.open = true;
|
|
1748
|
+
batch.openedAt = Date.now();
|
|
1749
|
+
batch.inTxn = 0;
|
|
1750
|
+
const timer = setTimeout(() => {
|
|
1751
|
+
batch.timer = null;
|
|
1752
|
+
// Fires only while the event loop is idle, i.e. between members.
|
|
1753
|
+
if (this.batch === batch && batch.open && this.db.inTransaction) {
|
|
1754
|
+
try {
|
|
1755
|
+
this.commitBatch(batch);
|
|
1756
|
+
}
|
|
1757
|
+
catch {
|
|
1758
|
+
/* retried at the next member boundary */
|
|
1759
|
+
}
|
|
1760
|
+
}
|
|
1761
|
+
}, batch.maxMs);
|
|
1762
|
+
timer.unref?.();
|
|
1763
|
+
batch.timer = timer;
|
|
1764
|
+
}
|
|
1765
|
+
/** After a member's writes: commit when the batch is full or old enough. */
|
|
1766
|
+
batchMemberDone() {
|
|
1767
|
+
const batch = this.batch;
|
|
1768
|
+
if (!batch)
|
|
1769
|
+
return;
|
|
1770
|
+
batch.stats.members += 1;
|
|
1771
|
+
if (!batch.open)
|
|
1772
|
+
return;
|
|
1773
|
+
if (!this.db.inTransaction) {
|
|
1774
|
+
// SQLite rolled back the batch (SQLITE_FULL / IOERR / NOMEM inside a
|
|
1775
|
+
// member's savepoint): every member since the last commit is lost.
|
|
1776
|
+
this.clearBatchTimer(batch);
|
|
1777
|
+
batch.open = false;
|
|
1778
|
+
throw new StorageError("batch transaction was rolled back by SQLite; members written since the last commit were lost");
|
|
1779
|
+
}
|
|
1780
|
+
batch.inTxn += 1;
|
|
1781
|
+
if (batch.inTxn >= batch.maxMembers || Date.now() - batch.openedAt >= batch.maxMs) {
|
|
1782
|
+
this.commitBatch(batch);
|
|
1783
|
+
}
|
|
1784
|
+
}
|
|
1785
|
+
clearBatchTimer(batch) {
|
|
1786
|
+
if (batch.timer) {
|
|
1787
|
+
clearTimeout(batch.timer);
|
|
1788
|
+
batch.timer = null;
|
|
1789
|
+
}
|
|
1790
|
+
}
|
|
1791
|
+
commitBatch(batch) {
|
|
1792
|
+
this.clearBatchTimer(batch);
|
|
1793
|
+
if (batch.open && this.db.inTransaction) {
|
|
1794
|
+
this.db.exec("COMMIT");
|
|
1795
|
+
batch.stats.commits += 1;
|
|
1796
|
+
}
|
|
1797
|
+
batch.open = false;
|
|
1798
|
+
}
|
|
1799
|
+
/** Close the batch window, committing the open transaction. No-op when closed. */
|
|
1800
|
+
endBatch() {
|
|
1801
|
+
const batch = this.batch;
|
|
1802
|
+
if (!batch)
|
|
1803
|
+
return;
|
|
1804
|
+
this.batch = null;
|
|
1805
|
+
this.commitBatch(batch);
|
|
1806
|
+
}
|
|
1807
|
+
/** Snapshot of the current batch window's counters (inactive zeros when closed). */
|
|
1808
|
+
batchStats() {
|
|
1809
|
+
if (this.batch)
|
|
1810
|
+
return { ...this.batch.stats };
|
|
1811
|
+
return { active: false, members: 0, commits: 0, beginFailures: 0 };
|
|
1812
|
+
}
|
|
1813
|
+
searchNodesFts(orgId, query, k, opts = {}) {
|
|
1814
|
+
if (k <= 0)
|
|
1815
|
+
return [];
|
|
1816
|
+
// Build an OR of the query's word tokens (recall-oriented). splitIdentifierWords
|
|
1817
|
+
// applies the SAME camelCase/snake_case splitting the index body uses, so
|
|
1818
|
+
// `AccountController` matches the indexed "account controller". Each token is
|
|
1819
|
+
// a quoted FTS5 string literal so punctuation/operators can't be injected
|
|
1820
|
+
// into the MATCH grammar.
|
|
1821
|
+
const tokens = Array.from(new Set(splitIdentifierWords(query).map((t) => t.toLowerCase()))).slice(0, 32);
|
|
1822
|
+
if (tokens.length === 0)
|
|
1823
|
+
return [];
|
|
1824
|
+
const match = tokens.map((t) => `"${t}"`).join(" OR ");
|
|
1825
|
+
try {
|
|
1826
|
+
const rows = this.db
|
|
1827
|
+
.prepare(`SELECT qualified_name AS qname, label
|
|
1828
|
+
FROM _sfgraph_node_fts
|
|
1829
|
+
WHERE org_id = ? AND body MATCH ?${opts.label !== undefined ? " AND lower(label) = lower(?)" : ""}
|
|
1830
|
+
ORDER BY bm25(_sfgraph_node_fts)
|
|
1831
|
+
LIMIT ?`)
|
|
1832
|
+
.all(...(opts.label !== undefined ? [orgId, match, opts.label, k] : [orgId, match, k]));
|
|
1833
|
+
return rows;
|
|
1834
|
+
}
|
|
1835
|
+
catch {
|
|
1836
|
+
return [];
|
|
1837
|
+
}
|
|
1838
|
+
}
|
|
1839
|
+
// ─── Task 11c: delta ingestion ────────────────────────────────────────────
|
|
1840
|
+
//
|
|
1841
|
+
// `_sfgraph_members` records, per member of a filesystem source, its key
|
|
1842
|
+
// (sourceUri), fingerprint (file sizes + mtimes), content hash and the
|
|
1843
|
+
// qnames it owns (nodes it emitted plus the sources of edges it emitted).
|
|
1844
|
+
// `_sfgraph_delta_pending` holds the qnames an in-flight delta is
|
|
1845
|
+
// rewriting; it is cleared when the delta completes, so a leftover row
|
|
1846
|
+
// means the previous delta was interrupted. `_sfgraph_member_generation`
|
|
1847
|
+
// records which sfgraph build wrote the rows: a delta under a different
|
|
1848
|
+
// build would mix old and new parser output, so it rebuilds instead. All
|
|
1849
|
+
// are created on first use (no schema-version bump: an older sfgraph simply
|
|
1850
|
+
// ignores them).
|
|
1851
|
+
deltaTablesReady = false;
|
|
1852
|
+
ensureDeltaTables() {
|
|
1853
|
+
if (this.deltaTablesReady)
|
|
1854
|
+
return;
|
|
1855
|
+
this.db.exec(`CREATE TABLE IF NOT EXISTS _sfgraph_members (
|
|
1856
|
+
org_id TEXT NOT NULL,
|
|
1857
|
+
member_key TEXT NOT NULL,
|
|
1858
|
+
fingerprint TEXT NOT NULL,
|
|
1859
|
+
hash TEXT NOT NULL,
|
|
1860
|
+
owned TEXT NOT NULL,
|
|
1861
|
+
PRIMARY KEY (org_id, member_key)
|
|
1862
|
+
);
|
|
1863
|
+
CREATE TABLE IF NOT EXISTS _sfgraph_member_generation (
|
|
1864
|
+
org_id TEXT PRIMARY KEY,
|
|
1865
|
+
generation TEXT NOT NULL
|
|
1866
|
+
);
|
|
1867
|
+
CREATE TABLE IF NOT EXISTS _sfgraph_delta_pending (
|
|
1868
|
+
org_id TEXT PRIMARY KEY,
|
|
1869
|
+
qnames TEXT NOT NULL,
|
|
1870
|
+
started_at INTEGER NOT NULL
|
|
1871
|
+
);`);
|
|
1872
|
+
this.deltaTablesReady = true;
|
|
1873
|
+
}
|
|
1874
|
+
listMemberRows(orgId) {
|
|
1875
|
+
this.ensureDeltaTables();
|
|
1876
|
+
const rows = this.stmt("SELECT member_key, fingerprint, hash, owned FROM _sfgraph_members WHERE org_id = ?").all(orgId);
|
|
1877
|
+
return rows.map((r) => {
|
|
1878
|
+
let owned = [];
|
|
1879
|
+
try {
|
|
1880
|
+
const v = JSON.parse(r.owned);
|
|
1881
|
+
if (Array.isArray(v))
|
|
1882
|
+
owned = v.filter((x) => typeof x === "string");
|
|
1883
|
+
}
|
|
1884
|
+
catch {
|
|
1885
|
+
owned = [];
|
|
1886
|
+
}
|
|
1887
|
+
return { key: r.member_key, fingerprint: r.fingerprint, hash: r.hash, owned };
|
|
1888
|
+
});
|
|
1889
|
+
}
|
|
1890
|
+
/**
|
|
1891
|
+
* Replace every member row of the org (a full ingest), stamped with the
|
|
1892
|
+
* `generation` (sfgraph build) whose parsers produced them.
|
|
1893
|
+
*/
|
|
1894
|
+
replaceMemberRows(orgId, rows, generation) {
|
|
1895
|
+
this.ensureDeltaTables();
|
|
1896
|
+
this.db.transaction(() => {
|
|
1897
|
+
this.stmt("DELETE FROM _sfgraph_members WHERE org_id = ?").run(orgId);
|
|
1898
|
+
this.writeMemberRows(orgId, rows);
|
|
1899
|
+
if (generation === undefined) {
|
|
1900
|
+
this.stmt("DELETE FROM _sfgraph_member_generation WHERE org_id = ?").run(orgId);
|
|
1901
|
+
}
|
|
1902
|
+
else {
|
|
1903
|
+
this.stmt(`INSERT INTO _sfgraph_member_generation(org_id, generation) VALUES (?, ?)
|
|
1904
|
+
ON CONFLICT(org_id) DO UPDATE SET generation = excluded.generation`).run(orgId, generation);
|
|
1905
|
+
}
|
|
1906
|
+
})();
|
|
1907
|
+
}
|
|
1908
|
+
/** The generation the org's member rows were written by, or null when unknown. */
|
|
1909
|
+
getMemberRowsGeneration(orgId) {
|
|
1910
|
+
this.ensureDeltaTables();
|
|
1911
|
+
const r = this.stmt("SELECT generation FROM _sfgraph_member_generation WHERE org_id = ?").get(orgId);
|
|
1912
|
+
return r?.generation ?? null;
|
|
1913
|
+
}
|
|
1914
|
+
upsertMemberRows(orgId, rows) {
|
|
1915
|
+
this.ensureDeltaTables();
|
|
1916
|
+
this.db.transaction(() => this.writeMemberRows(orgId, rows))();
|
|
1917
|
+
}
|
|
1918
|
+
writeMemberRows(orgId, rows) {
|
|
1919
|
+
const st = this.stmt(`INSERT INTO _sfgraph_members(org_id, member_key, fingerprint, hash, owned) VALUES (?, ?, ?, ?, ?)
|
|
1920
|
+
ON CONFLICT(org_id, member_key) DO UPDATE SET fingerprint = excluded.fingerprint,
|
|
1921
|
+
hash = excluded.hash, owned = excluded.owned`);
|
|
1922
|
+
for (const r of rows)
|
|
1923
|
+
st.run(orgId, r.key, r.fingerprint, r.hash, JSON.stringify(r.owned));
|
|
1924
|
+
}
|
|
1925
|
+
deleteMemberRows(orgId, keys) {
|
|
1926
|
+
this.ensureDeltaTables();
|
|
1927
|
+
const st = this.stmt("DELETE FROM _sfgraph_members WHERE org_id = ? AND member_key = ?");
|
|
1928
|
+
this.db.transaction(() => {
|
|
1929
|
+
for (const k of keys)
|
|
1930
|
+
st.run(orgId, k);
|
|
1931
|
+
})();
|
|
1932
|
+
}
|
|
1933
|
+
clearMemberRows(orgId) {
|
|
1934
|
+
this.ensureDeltaTables();
|
|
1935
|
+
// Rows and their generation stamp go together (final review core M-4).
|
|
1936
|
+
this.db.transaction(() => {
|
|
1937
|
+
this.stmt("DELETE FROM _sfgraph_members WHERE org_id = ?").run(orgId);
|
|
1938
|
+
this.stmt("DELETE FROM _sfgraph_member_generation WHERE org_id = ?").run(orgId);
|
|
1939
|
+
})();
|
|
1940
|
+
}
|
|
1941
|
+
getDeltaPending(orgId) {
|
|
1942
|
+
this.ensureDeltaTables();
|
|
1943
|
+
const row = this.stmt("SELECT qnames FROM _sfgraph_delta_pending WHERE org_id = ?").get(orgId);
|
|
1944
|
+
if (!row)
|
|
1945
|
+
return null;
|
|
1946
|
+
try {
|
|
1947
|
+
const v = JSON.parse(row.qnames);
|
|
1948
|
+
return Array.isArray(v) ? v.filter((x) => typeof x === "string") : [];
|
|
1949
|
+
}
|
|
1950
|
+
catch {
|
|
1951
|
+
return [];
|
|
1952
|
+
}
|
|
1953
|
+
}
|
|
1954
|
+
setDeltaPending(orgId, qnames) {
|
|
1955
|
+
this.ensureDeltaTables();
|
|
1956
|
+
this.stmt(`INSERT INTO _sfgraph_delta_pending(org_id, qnames, started_at) VALUES (?, ?, ?)
|
|
1957
|
+
ON CONFLICT(org_id) DO UPDATE SET qnames = excluded.qnames, started_at = excluded.started_at`).run(orgId, JSON.stringify(qnames), Date.now());
|
|
1958
|
+
}
|
|
1959
|
+
clearDeltaPending(orgId) {
|
|
1960
|
+
this.ensureDeltaTables();
|
|
1961
|
+
this.stmt("DELETE FROM _sfgraph_delta_pending WHERE org_id = ?").run(orgId);
|
|
1962
|
+
}
|
|
1963
|
+
/** The subset of `qnames` that exist as nodes (stored spelling). */
|
|
1964
|
+
existingQnames(orgId, qnames) {
|
|
1965
|
+
const out = new Set();
|
|
1966
|
+
for (const slice of chunks([...qnames], 400)) {
|
|
1967
|
+
const rows = this.stmt(`SELECT qualified_name FROM _sfgraph_node_index WHERE org_id = ? AND qualified_name IN (${slice.map(() => "?").join(",")})`).all(orgId, ...slice);
|
|
1968
|
+
for (const r of rows)
|
|
1969
|
+
out.add(r.qualified_name);
|
|
1970
|
+
}
|
|
1971
|
+
return out;
|
|
1972
|
+
}
|
|
1973
|
+
/** Stored spellings of the nodes matching `qnames` case-insensitively. */
|
|
1974
|
+
existingQnamesNocase(orgId, qnames) {
|
|
1975
|
+
const out = new Set();
|
|
1976
|
+
for (const slice of chunks([...qnames], 400)) {
|
|
1977
|
+
const rows = this.stmt(`SELECT qualified_name FROM _sfgraph_node_index WHERE org_id = ? AND qualified_name COLLATE NOCASE IN (${slice.map(() => "?").join(",")})`).all(orgId, ...slice);
|
|
1978
|
+
for (const r of rows)
|
|
1979
|
+
out.add(r.qualified_name);
|
|
1980
|
+
}
|
|
1981
|
+
return out;
|
|
1982
|
+
}
|
|
1983
|
+
/** Overwrite the source_hash of existing nodes (delta's stale sentinel). */
|
|
1984
|
+
setNodeSourceHash(orgId, qnames, hash) {
|
|
1985
|
+
this.syncCaches();
|
|
1986
|
+
let n = 0;
|
|
1987
|
+
this.db.transaction(() => {
|
|
1988
|
+
for (const slice of chunks([...qnames], 400)) {
|
|
1989
|
+
const rows = this.stmt(`SELECT qualified_name, label FROM _sfgraph_node_index WHERE org_id = ? AND qualified_name IN (${slice.map(() => "?").join(",")})`).all(orgId, ...slice);
|
|
1990
|
+
for (const r of rows) {
|
|
1991
|
+
const tbl = this.nodeLabelCache.get(r.label);
|
|
1992
|
+
if (!tbl)
|
|
1993
|
+
continue;
|
|
1994
|
+
n += this.stmt(`UPDATE ${tbl} SET source_hash = ? WHERE org_id = ? AND qualified_name = ?`).run(hash, orgId, r.qualified_name).changes;
|
|
1995
|
+
}
|
|
1996
|
+
}
|
|
1997
|
+
})();
|
|
1998
|
+
return n;
|
|
1999
|
+
}
|
|
2000
|
+
/** source_hash and last_modified_at of existing nodes, keyed by stored qname. */
|
|
2001
|
+
getNodeStamps(orgId, qnames) {
|
|
2002
|
+
this.syncCaches();
|
|
2003
|
+
const out = new Map();
|
|
2004
|
+
for (const slice of chunks([...qnames], 400)) {
|
|
2005
|
+
const rows = this.stmt(`SELECT qualified_name, label FROM _sfgraph_node_index WHERE org_id = ? AND qualified_name IN (${slice.map(() => "?").join(",")})`).all(orgId, ...slice);
|
|
2006
|
+
for (const r of rows) {
|
|
2007
|
+
const tbl = this.nodeLabelCache.get(r.label);
|
|
2008
|
+
if (!tbl)
|
|
2009
|
+
continue;
|
|
2010
|
+
const row = this.stmt(`SELECT source_hash AS h, last_modified_at AS m FROM ${tbl} WHERE org_id = ? AND qualified_name = ?`).get(orgId, r.qualified_name);
|
|
2011
|
+
if (row)
|
|
2012
|
+
out.set(r.qualified_name, { hash: row.h, lastModifiedAt: row.m });
|
|
2013
|
+
}
|
|
2014
|
+
}
|
|
2015
|
+
return out;
|
|
2016
|
+
}
|
|
2017
|
+
/**
|
|
2018
|
+
* Put recorded stamps back. `when: "hash-is"` restores both fields on nodes
|
|
2019
|
+
* whose source_hash is currently `hash` (a sentinel nobody re-emitted);
|
|
2020
|
+
* `when: "unchanged"` restores last_modified_at on nodes whose source_hash
|
|
2021
|
+
* equals the recorded one (re-merged with identical content).
|
|
2022
|
+
*/
|
|
2023
|
+
restoreNodeStamps(orgId, stamps, cond) {
|
|
2024
|
+
this.syncCaches();
|
|
2025
|
+
let n = 0;
|
|
2026
|
+
this.db.transaction(() => {
|
|
2027
|
+
for (const slice of chunks([...stamps.keys()], 400)) {
|
|
2028
|
+
const rows = this.stmt(`SELECT qualified_name, label FROM _sfgraph_node_index WHERE org_id = ? AND qualified_name IN (${slice.map(() => "?").join(",")})`).all(orgId, ...slice);
|
|
2029
|
+
for (const r of rows) {
|
|
2030
|
+
const tbl = this.nodeLabelCache.get(r.label);
|
|
2031
|
+
const st = stamps.get(r.qualified_name);
|
|
2032
|
+
if (!tbl || !st)
|
|
2033
|
+
continue;
|
|
2034
|
+
n +=
|
|
2035
|
+
cond.when === "hash-is"
|
|
2036
|
+
? this.stmt(`UPDATE ${tbl} SET source_hash = ?, last_modified_at = ? WHERE org_id = ? AND qualified_name = ? AND source_hash = ?`).run(st.hash, st.lastModifiedAt, orgId, r.qualified_name, cond.hash).changes
|
|
2037
|
+
: this.stmt(`UPDATE ${tbl} SET last_modified_at = ? WHERE org_id = ? AND qualified_name = ? AND source_hash = ? AND last_modified_at <> ?`).run(st.lastModifiedAt, orgId, r.qualified_name, st.hash, st.lastModifiedAt)
|
|
2038
|
+
.changes;
|
|
2039
|
+
}
|
|
2040
|
+
}
|
|
2041
|
+
})();
|
|
2042
|
+
return n;
|
|
2043
|
+
}
|
|
2044
|
+
listNodesWithSourceHash(orgId, hash) {
|
|
2045
|
+
this.syncCaches();
|
|
2046
|
+
const out = [];
|
|
2047
|
+
for (const tbl of this.nodeLabelCache.values()) {
|
|
2048
|
+
const rows = this.stmt(`SELECT qualified_name FROM ${tbl} WHERE org_id = ? AND source_hash = ?`).all(orgId, hash);
|
|
2049
|
+
for (const r of rows)
|
|
2050
|
+
out.push(r.qualified_name);
|
|
2051
|
+
}
|
|
2052
|
+
return out;
|
|
2053
|
+
}
|
|
2054
|
+
/** Delete every outgoing edge of `srcs` (any pass); returns what was deleted. */
|
|
2055
|
+
deleteEdgesFrom(orgId, srcs) {
|
|
2056
|
+
this.syncCaches();
|
|
2057
|
+
const out = [];
|
|
2058
|
+
const all = [...srcs];
|
|
2059
|
+
if (all.length === 0)
|
|
2060
|
+
return out;
|
|
2061
|
+
this.db.transaction(() => {
|
|
2062
|
+
for (const [relType, tbl] of this.edgeRelCache) {
|
|
2063
|
+
for (const slice of chunks(all, 400)) {
|
|
2064
|
+
const ph = slice.map(() => "?").join(",");
|
|
2065
|
+
const rows = this.stmt(`SELECT src_qname, dst_qname, first_seen_at FROM ${tbl} WHERE org_id = ? AND src_qname IN (${ph})`).all(orgId, ...slice);
|
|
2066
|
+
if (rows.length === 0)
|
|
2067
|
+
continue;
|
|
2068
|
+
this.stmt(`DELETE FROM ${tbl} WHERE org_id = ? AND src_qname IN (${ph})`).run(orgId, ...slice);
|
|
2069
|
+
for (const r of rows) {
|
|
2070
|
+
out.push({ relType, src: r.src_qname, dst: r.dst_qname, firstSeenAt: r.first_seen_at });
|
|
2071
|
+
}
|
|
2072
|
+
}
|
|
2073
|
+
}
|
|
2074
|
+
})();
|
|
2075
|
+
return out;
|
|
2076
|
+
}
|
|
2077
|
+
/** Edges (any rel type) whose dst is one of `dsts`. */
|
|
2078
|
+
listEdgesToAny(orgId, dsts) {
|
|
2079
|
+
this.syncCaches();
|
|
2080
|
+
const out = [];
|
|
2081
|
+
const all = [...dsts];
|
|
2082
|
+
if (all.length === 0)
|
|
2083
|
+
return out;
|
|
2084
|
+
for (const [relType, tbl] of this.edgeRelCache) {
|
|
2085
|
+
for (const slice of chunks(all, 400)) {
|
|
2086
|
+
const rows = this.stmt(`SELECT org_id, src_qname, dst_qname, attributes, first_seen_at, last_seen_at FROM ${tbl} WHERE org_id = ? AND dst_qname IN (${slice.map(() => "?").join(",")})`).all(orgId, ...slice);
|
|
2087
|
+
for (const r of rows)
|
|
2088
|
+
out.push(rowToEdge(r, relType));
|
|
2089
|
+
}
|
|
2090
|
+
}
|
|
2091
|
+
return out;
|
|
2092
|
+
}
|
|
2093
|
+
/** Edges of `relType` whose src is one of `srcs` (NOCASE, like listEdgesFrom). */
|
|
2094
|
+
listEdgesFromAny(orgId, srcs, relType) {
|
|
2095
|
+
this.syncCaches();
|
|
2096
|
+
const out = [];
|
|
2097
|
+
const tbl = this.edgeRelCache.get(relType);
|
|
2098
|
+
const all = [...srcs];
|
|
2099
|
+
if (!tbl || all.length === 0)
|
|
2100
|
+
return out;
|
|
2101
|
+
for (const slice of chunks(all, 400)) {
|
|
2102
|
+
const rows = this.stmt(`SELECT org_id, src_qname, dst_qname, attributes, first_seen_at, last_seen_at FROM ${tbl} WHERE org_id = ? AND src_qname IN (${slice.map(() => "?").join(",")})`).all(orgId, ...slice);
|
|
2103
|
+
for (const r of rows)
|
|
2104
|
+
out.push(rowToEdge(r, relType));
|
|
2105
|
+
}
|
|
2106
|
+
return out;
|
|
2107
|
+
}
|
|
2108
|
+
/**
|
|
2109
|
+
* Whether any node of `labels` has attributes JSON text containing one of
|
|
2110
|
+
* `needles` (a plain substring test, so a superset of "has that key").
|
|
2111
|
+
* Reads the text only, no JSON parse; stops at the first hit.
|
|
2112
|
+
*/
|
|
2113
|
+
anyNodeAttributesContain(orgId, labels, needles) {
|
|
2114
|
+
this.syncCaches();
|
|
2115
|
+
if (needles.length === 0)
|
|
2116
|
+
return false;
|
|
2117
|
+
const cond = needles.map(() => "instr(attributes, ?) > 0").join(" OR ");
|
|
2118
|
+
for (const label of labels) {
|
|
2119
|
+
const tbl = this.nodeLabelCache.get(label);
|
|
2120
|
+
if (!tbl)
|
|
2121
|
+
continue;
|
|
2122
|
+
if (this.stmt(`SELECT 1 FROM ${tbl} WHERE org_id = ? AND (${cond}) LIMIT 1`).get(orgId, ...needles))
|
|
2123
|
+
return true;
|
|
2124
|
+
}
|
|
2125
|
+
return false;
|
|
2126
|
+
}
|
|
2127
|
+
/** `anyNodeAttributesContain` for the edges of `relTypes`. */
|
|
2128
|
+
anyEdgeAttributesContain(orgId, relTypes, needle) {
|
|
2129
|
+
this.syncCaches();
|
|
2130
|
+
for (const rel of relTypes) {
|
|
2131
|
+
const tbl = this.edgeRelCache.get(rel);
|
|
2132
|
+
if (!tbl)
|
|
2133
|
+
continue;
|
|
2134
|
+
if (this.stmt(`SELECT 1 FROM ${tbl} WHERE org_id = ? AND instr(attributes, ?) > 0 LIMIT 1`).get(orgId, needle))
|
|
2135
|
+
return true;
|
|
2136
|
+
}
|
|
2137
|
+
return false;
|
|
2138
|
+
}
|
|
2139
|
+
/**
|
|
2140
|
+
* Sum of `attributes[attr]` over the nodes of `labels` where it is a
|
|
2141
|
+
* number > 0, in listNodesByLabel order (label by label).
|
|
2142
|
+
*/
|
|
2143
|
+
sumPositiveNodeAttribute(orgId, labels, attr) {
|
|
2144
|
+
this.syncCaches();
|
|
2145
|
+
const path = `$.${JSON.stringify(attr)}`;
|
|
2146
|
+
let sum = 0;
|
|
2147
|
+
for (const label of labels) {
|
|
2148
|
+
const tbl = this.nodeLabelCache.get(label);
|
|
2149
|
+
if (!tbl)
|
|
2150
|
+
continue;
|
|
2151
|
+
const rows = this.stmt(`SELECT json_type(attributes, ?) AS t, json_extract(attributes, ?) AS v FROM ${tbl} WHERE org_id = ? AND instr(attributes, ?) > 0`).all(path, path, orgId, JSON.stringify(attr));
|
|
2152
|
+
for (const r of rows) {
|
|
2153
|
+
if ((r.t === "integer" || r.t === "real") && typeof r.v === "number" && r.v > 0)
|
|
2154
|
+
sum += r.v;
|
|
2155
|
+
}
|
|
2156
|
+
}
|
|
2157
|
+
return sum;
|
|
2158
|
+
}
|
|
2159
|
+
/**
|
|
2160
|
+
* Stored qnames (as `_sfgraph_node_index` spells them) of the org's nodes
|
|
2161
|
+
* whose `attributes[attr]` is JSON `true` — exactly the nodes for which
|
|
2162
|
+
* `getNode(q)?.attributes[attr] === true`, read in one pass per label.
|
|
2163
|
+
*/
|
|
2164
|
+
listNodesWithTrueAttribute(orgId, attr) {
|
|
2165
|
+
this.syncCaches();
|
|
2166
|
+
const out = new Set();
|
|
2167
|
+
const path = `$.${JSON.stringify(attr)}`;
|
|
2168
|
+
for (const [label, tbl] of this.nodeLabelCache) {
|
|
2169
|
+
const rows = this.stmt(`SELECT i.qualified_name AS q FROM _sfgraph_node_index i
|
|
2170
|
+
JOIN ${tbl} t ON t.org_id = i.org_id AND t.qualified_name = i.qualified_name
|
|
2171
|
+
WHERE i.org_id = ? AND i.label = ? AND json_type(t.attributes, ?) = 'true'`).all(orgId, label, path);
|
|
2172
|
+
for (const r of rows)
|
|
2173
|
+
out.add(r.q);
|
|
2174
|
+
}
|
|
2175
|
+
return out;
|
|
2176
|
+
}
|
|
2177
|
+
/**
|
|
2178
|
+
* Delete reflection-walker edges (`source: "reflection"` REFERENCES) of the
|
|
2179
|
+
* org: those FROM `srcs` and/or INTO `dsts`; with neither, all of them.
|
|
2180
|
+
*/
|
|
2181
|
+
deleteReflectionEdges(orgId, scope = {}) {
|
|
2182
|
+
this.syncCaches();
|
|
2183
|
+
const tbl = this.edgeRelCache.get("REFERENCES");
|
|
2184
|
+
if (!tbl)
|
|
2185
|
+
return 0;
|
|
2186
|
+
const isRefl = "json_extract(attributes, '$.source') = 'reflection'";
|
|
2187
|
+
let n = 0;
|
|
2188
|
+
this.db.transaction(() => {
|
|
2189
|
+
if (!scope.srcs && !scope.dsts) {
|
|
2190
|
+
n += this.stmt(`DELETE FROM ${tbl} WHERE org_id = ? AND ${isRefl}`).run(orgId).changes;
|
|
2191
|
+
return;
|
|
2192
|
+
}
|
|
2193
|
+
for (const [col, list] of [
|
|
2194
|
+
["src_qname", scope.srcs],
|
|
2195
|
+
["dst_qname", scope.dsts],
|
|
2196
|
+
]) {
|
|
2197
|
+
if (!list)
|
|
2198
|
+
continue;
|
|
2199
|
+
for (const slice of chunks([...list], 400)) {
|
|
2200
|
+
n += this.stmt(`DELETE FROM ${tbl} WHERE org_id = ? AND ${col} IN (${slice.map(() => "?").join(",")}) AND ${isRefl}`).run(orgId, ...slice).changes;
|
|
2201
|
+
}
|
|
2202
|
+
}
|
|
2203
|
+
})();
|
|
2204
|
+
return n;
|
|
2205
|
+
}
|
|
2206
|
+
/**
|
|
2207
|
+
* Qnames of nodes whose attributes JSON contains one of `needles` as a whole
|
|
2208
|
+
* JSON string (case-insensitive for ASCII). A superset of the nodes whose
|
|
2209
|
+
* attribute string values equal a needle — the reflection walker's input.
|
|
2210
|
+
*/
|
|
2211
|
+
findNodesMentioning(orgId, needles) {
|
|
2212
|
+
this.syncCaches();
|
|
2213
|
+
const out = new Set();
|
|
2214
|
+
if (needles.length === 0)
|
|
2215
|
+
return out;
|
|
2216
|
+
const likeEsc = (s) => s.replace(/[\\%_]/g, (c) => `\\${c}`);
|
|
2217
|
+
const pats = needles.map((nd) => `%${likeEsc(JSON.stringify(nd))}%`);
|
|
2218
|
+
for (const tbl of this.nodeLabelCache.values()) {
|
|
2219
|
+
for (const slice of chunks(pats, 50)) {
|
|
2220
|
+
const rows = this.stmt(`SELECT qualified_name FROM ${tbl} WHERE org_id = ? AND (${slice.map(() => "attributes LIKE ? ESCAPE '\\'").join(" OR ")})`).all(orgId, ...slice);
|
|
2221
|
+
for (const r of rows)
|
|
2222
|
+
out.add(r.qualified_name);
|
|
2223
|
+
}
|
|
2224
|
+
}
|
|
2225
|
+
return out;
|
|
2226
|
+
}
|
|
2227
|
+
/** `src|dst` (lower-cased) of every resolver-emitted (`resolvedBy`) REFERENCES edge. */
|
|
2228
|
+
listResolverReferencePairs(orgId) {
|
|
2229
|
+
this.syncCaches();
|
|
2230
|
+
const out = new Set();
|
|
2231
|
+
const tbl = this.edgeRelCache.get("REFERENCES");
|
|
2232
|
+
if (!tbl)
|
|
2233
|
+
return out;
|
|
2234
|
+
const rows = this.stmt(`SELECT src_qname, dst_qname FROM ${tbl} WHERE org_id = ? AND json_extract(attributes, '$.resolvedBy') IS NOT NULL`).all(orgId);
|
|
2235
|
+
for (const r of rows)
|
|
2236
|
+
out.add(`${r.src_qname}${KSEP}${r.dst_qname}`.toLowerCase());
|
|
2237
|
+
return out;
|
|
2238
|
+
}
|
|
2239
|
+
/** `src|dst` (lower-cased) of every CANONICAL_OF edge. */
|
|
2240
|
+
listCanonicalPairs(orgId) {
|
|
2241
|
+
this.syncCaches();
|
|
2242
|
+
const out = new Set();
|
|
2243
|
+
const tbl = this.edgeRelCache.get("CANONICAL_OF");
|
|
2244
|
+
if (!tbl)
|
|
2245
|
+
return out;
|
|
2246
|
+
const rows = this.stmt(`SELECT src_qname, dst_qname FROM ${tbl} WHERE org_id = ?`).all(orgId);
|
|
2247
|
+
for (const r of rows)
|
|
2248
|
+
out.add(`${r.src_qname}${KSEP}${r.dst_qname}`.toLowerCase());
|
|
2249
|
+
return out;
|
|
2250
|
+
}
|
|
2251
|
+
/** Lower an edge's first_seen_at to `firstSeenAt` (re-created edges keep their history). */
|
|
2252
|
+
restoreEdgeFirstSeen(orgId, edges) {
|
|
2253
|
+
this.syncCaches();
|
|
2254
|
+
this.db.transaction(() => {
|
|
2255
|
+
for (const e of edges) {
|
|
2256
|
+
const tbl = this.edgeRelCache.get(e.relType);
|
|
2257
|
+
if (!tbl)
|
|
2258
|
+
continue;
|
|
2259
|
+
this.stmt(`UPDATE ${tbl} SET first_seen_at = ? WHERE org_id = ? AND src_qname = ? AND dst_qname = ? AND first_seen_at > ?`).run(e.firstSeenAt, orgId, e.src, e.dst, e.firstSeenAt);
|
|
2260
|
+
}
|
|
2261
|
+
})();
|
|
2262
|
+
}
|
|
657
2263
|
// Helpers for shared usage
|
|
658
2264
|
getNodeLabelTables() {
|
|
2265
|
+
this.syncCaches();
|
|
659
2266
|
return new Map(this.nodeLabelCache);
|
|
660
2267
|
}
|
|
661
2268
|
getEdgeRelTables() {
|
|
2269
|
+
this.syncCaches();
|
|
662
2270
|
return new Map(this.edgeRelCache);
|
|
663
2271
|
}
|
|
664
2272
|
// For tests/diagnostics
|
|
665
2273
|
_explainReverseEdgeQuery(relType) {
|
|
2274
|
+
this.syncCaches();
|
|
666
2275
|
const tbl = this.edgeRelCache.get(relType);
|
|
667
2276
|
if (!tbl)
|
|
668
2277
|
throw new StorageError(`unknown rel type ${relType}`);
|