@ryanstark24/sfgraph-core 1.2.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/analyze/apex-platform.d.ts +23 -0
- package/dist/analyze/apex-platform.d.ts.map +1 -0
- package/dist/analyze/apex-platform.js +112 -0
- package/dist/analyze/apex-platform.js.map +1 -0
- package/dist/analyze/audit-graph.d.ts +51 -3
- package/dist/analyze/audit-graph.d.ts.map +1 -1
- package/dist/analyze/audit-graph.js +382 -14
- package/dist/analyze/audit-graph.js.map +1 -1
- package/dist/analyze/component-scope.d.ts +82 -0
- package/dist/analyze/component-scope.d.ts.map +1 -0
- package/dist/analyze/component-scope.js +207 -0
- package/dist/analyze/component-scope.js.map +1 -0
- package/dist/analyze/containment.d.ts +91 -0
- package/dist/analyze/containment.d.ts.map +1 -0
- package/dist/analyze/containment.js +175 -0
- package/dist/analyze/containment.js.map +1 -0
- package/dist/analyze/cross-org.d.ts +35 -0
- package/dist/analyze/cross-org.d.ts.map +1 -1
- package/dist/analyze/cross-org.js +102 -3
- package/dist/analyze/cross-org.js.map +1 -1
- package/dist/analyze/dead-code.d.ts +99 -3
- package/dist/analyze/dead-code.d.ts.map +1 -1
- package/dist/analyze/dead-code.js +282 -10
- package/dist/analyze/dead-code.js.map +1 -1
- package/dist/analyze/dependencies.d.ts +21 -2
- package/dist/analyze/dependencies.d.ts.map +1 -1
- package/dist/analyze/dependencies.js +102 -12
- package/dist/analyze/dependencies.js.map +1 -1
- package/dist/analyze/dependents.d.ts +116 -1
- package/dist/analyze/dependents.d.ts.map +1 -1
- package/dist/analyze/dependents.js +285 -13
- package/dist/analyze/dependents.js.map +1 -1
- package/dist/analyze/deployment-manifest/generate.d.ts +24 -0
- package/dist/analyze/deployment-manifest/generate.d.ts.map +1 -1
- package/dist/analyze/deployment-manifest/generate.js +40 -12
- package/dist/analyze/deployment-manifest/generate.js.map +1 -1
- package/dist/analyze/deployment-manifest/member-name-formatters.d.ts.map +1 -1
- package/dist/analyze/deployment-manifest/member-name-formatters.js +1 -0
- package/dist/analyze/deployment-manifest/member-name-formatters.js.map +1 -1
- package/dist/analyze/deployment-manifest/member-types.d.ts.map +1 -1
- package/dist/analyze/deployment-manifest/member-types.js +2 -0
- package/dist/analyze/deployment-manifest/member-types.js.map +1 -1
- package/dist/analyze/diff-seeds.d.ts +33 -0
- package/dist/analyze/diff-seeds.d.ts.map +1 -0
- package/dist/analyze/diff-seeds.js +299 -0
- package/dist/analyze/diff-seeds.js.map +1 -0
- package/dist/analyze/edge-filters.d.ts +62 -0
- package/dist/analyze/edge-filters.d.ts.map +1 -0
- package/dist/analyze/edge-filters.js +127 -0
- package/dist/analyze/edge-filters.js.map +1 -0
- package/dist/analyze/field-scope.d.ts +25 -0
- package/dist/analyze/field-scope.d.ts.map +1 -0
- package/dist/analyze/field-scope.js +94 -0
- package/dist/analyze/field-scope.js.map +1 -0
- package/dist/analyze/find-nodes.d.ts +48 -12
- package/dist/analyze/find-nodes.d.ts.map +1 -1
- package/dist/analyze/find-nodes.js +323 -12
- package/dist/analyze/find-nodes.js.map +1 -1
- package/dist/analyze/findings.d.ts +1 -1
- package/dist/analyze/findings.d.ts.map +1 -1
- package/dist/analyze/findings.js +15 -3
- package/dist/analyze/findings.js.map +1 -1
- package/dist/analyze/git-diff.d.ts +27 -1
- package/dist/analyze/git-diff.d.ts.map +1 -1
- package/dist/analyze/git-diff.js +198 -26
- package/dist/analyze/git-diff.js.map +1 -1
- package/dist/analyze/governor.d.ts +25 -4
- package/dist/analyze/governor.d.ts.map +1 -1
- package/dist/analyze/governor.js +82 -18
- package/dist/analyze/governor.js.map +1 -1
- package/dist/analyze/index.d.ts +6 -0
- package/dist/analyze/index.d.ts.map +1 -1
- package/dist/analyze/index.js +6 -0
- package/dist/analyze/index.js.map +1 -1
- package/dist/analyze/path-to-qname.d.ts +9 -0
- package/dist/analyze/path-to-qname.d.ts.map +1 -1
- package/dist/analyze/path-to-qname.js +58 -15
- package/dist/analyze/path-to-qname.js.map +1 -1
- package/dist/analyze/populate.d.ts +40 -0
- package/dist/analyze/populate.d.ts.map +1 -1
- package/dist/analyze/populate.js +203 -58
- package/dist/analyze/populate.js.map +1 -1
- package/dist/analyze/pseudo-endpoints.d.ts +67 -0
- package/dist/analyze/pseudo-endpoints.d.ts.map +1 -0
- package/dist/analyze/pseudo-endpoints.js +78 -0
- package/dist/analyze/pseudo-endpoints.js.map +1 -0
- package/dist/analyze/security.d.ts +17 -0
- package/dist/analyze/security.d.ts.map +1 -1
- package/dist/analyze/security.js +103 -18
- package/dist/analyze/security.js.map +1 -1
- package/dist/analyze/test-coverage.d.ts +48 -0
- package/dist/analyze/test-coverage.d.ts.map +1 -1
- package/dist/analyze/test-coverage.js +72 -1
- package/dist/analyze/test-coverage.js.map +1 -1
- package/dist/analyze/wip-changed-files.d.ts +88 -0
- package/dist/analyze/wip-changed-files.d.ts.map +1 -0
- package/dist/analyze/wip-changed-files.js +422 -0
- package/dist/analyze/wip-changed-files.js.map +1 -0
- package/dist/analyze/wip-impact.d.ts +59 -0
- package/dist/analyze/wip-impact.d.ts.map +1 -1
- package/dist/analyze/wip-impact.js +199 -116
- package/dist/analyze/wip-impact.js.map +1 -1
- package/dist/domain/apex-platform.d.ts +10 -0
- package/dist/domain/apex-platform.d.ts.map +1 -0
- package/dist/domain/apex-platform.js +231 -0
- package/dist/domain/apex-platform.js.map +1 -0
- package/dist/domain/index.d.ts +2 -0
- package/dist/domain/index.d.ts.map +1 -1
- package/dist/domain/index.js +2 -0
- package/dist/domain/index.js.map +1 -1
- package/dist/domain/metadata-category.d.ts +1 -0
- package/dist/domain/metadata-category.d.ts.map +1 -1
- package/dist/domain/metadata-category.js +1 -0
- package/dist/domain/metadata-category.js.map +1 -1
- package/dist/domain/rel-types.d.ts +7 -0
- package/dist/domain/rel-types.d.ts.map +1 -1
- package/dist/domain/rel-types.js +7 -0
- package/dist/domain/rel-types.js.map +1 -1
- package/dist/domain/standard-sobjects.d.ts +20 -0
- package/dist/domain/standard-sobjects.d.ts.map +1 -0
- package/dist/domain/standard-sobjects.js +183 -0
- package/dist/domain/standard-sobjects.js.map +1 -0
- package/dist/embedding/cache.d.ts +41 -0
- package/dist/embedding/cache.d.ts.map +1 -0
- package/dist/embedding/cache.js +69 -0
- package/dist/embedding/cache.js.map +1 -0
- package/dist/embedding/embed.d.ts +59 -0
- package/dist/embedding/embed.d.ts.map +1 -1
- package/dist/embedding/embed.js +207 -29
- package/dist/embedding/embed.js.map +1 -1
- package/dist/embedding/index.d.ts +7 -0
- package/dist/embedding/index.d.ts.map +1 -1
- package/dist/embedding/index.js +4 -0
- package/dist/embedding/index.js.map +1 -1
- package/dist/embedding/pool.d.ts +66 -6
- package/dist/embedding/pool.d.ts.map +1 -1
- package/dist/embedding/pool.js +156 -6
- package/dist/embedding/pool.js.map +1 -1
- package/dist/embedding/queue.d.ts +166 -9
- package/dist/embedding/queue.d.ts.map +1 -1
- package/dist/embedding/queue.js +397 -39
- package/dist/embedding/queue.js.map +1 -1
- package/dist/embedding/tokenize.d.ts +18 -0
- package/dist/embedding/tokenize.d.ts.map +1 -0
- package/dist/embedding/tokenize.js +43 -0
- package/dist/embedding/tokenize.js.map +1 -0
- package/dist/embedding/worker.d.ts +0 -5
- package/dist/embedding/worker.d.ts.map +1 -1
- package/dist/embedding/worker.js +27 -4
- package/dist/embedding/worker.js.map +1 -1
- package/dist/extractors/filesystem/index.d.ts +21 -1
- package/dist/extractors/filesystem/index.d.ts.map +1 -1
- package/dist/extractors/filesystem/index.js +195 -38
- package/dist/extractors/filesystem/index.js.map +1 -1
- package/dist/extractors/filesystem/vlocity-datapacks.d.ts +27 -0
- package/dist/extractors/filesystem/vlocity-datapacks.d.ts.map +1 -0
- package/dist/extractors/filesystem/vlocity-datapacks.js +201 -0
- package/dist/extractors/filesystem/vlocity-datapacks.js.map +1 -0
- package/dist/extractors/interfaces/metadata-source.d.ts +40 -0
- package/dist/extractors/interfaces/metadata-source.d.ts.map +1 -1
- package/dist/extractors/live-org/bulk-retrieve.d.ts +206 -1
- package/dist/extractors/live-org/bulk-retrieve.d.ts.map +1 -1
- package/dist/extractors/live-org/bulk-retrieve.js +706 -94
- package/dist/extractors/live-org/bulk-retrieve.js.map +1 -1
- package/dist/extractors/live-org/capabilities.d.ts +15 -0
- package/dist/extractors/live-org/capabilities.d.ts.map +1 -1
- package/dist/extractors/live-org/capabilities.js +54 -9
- package/dist/extractors/live-org/capabilities.js.map +1 -1
- package/dist/extractors/live-org/extractors/apex.d.ts +12 -2
- package/dist/extractors/live-org/extractors/apex.d.ts.map +1 -1
- package/dist/extractors/live-org/extractors/apex.js +171 -75
- package/dist/extractors/live-org/extractors/apex.js.map +1 -1
- package/dist/extractors/live-org/extractors/custom-settings.d.ts +4 -0
- package/dist/extractors/live-org/extractors/custom-settings.d.ts.map +1 -0
- package/dist/extractors/live-org/extractors/custom-settings.js +76 -0
- package/dist/extractors/live-org/extractors/custom-settings.js.map +1 -0
- package/dist/extractors/live-org/extractors/flow.d.ts +2 -1
- package/dist/extractors/live-org/extractors/flow.d.ts.map +1 -1
- package/dist/extractors/live-org/extractors/flow.js +12 -3
- package/dist/extractors/live-org/extractors/flow.js.map +1 -1
- package/dist/extractors/live-org/extractors/generic-metadata.d.ts +23 -3
- package/dist/extractors/live-org/extractors/generic-metadata.d.ts.map +1 -1
- package/dist/extractors/live-org/extractors/generic-metadata.js +110 -25
- package/dist/extractors/live-org/extractors/generic-metadata.js.map +1 -1
- package/dist/extractors/live-org/extractors/generic-sobject.js.map +1 -1
- package/dist/extractors/live-org/extractors/integration.d.ts +2 -1
- package/dist/extractors/live-org/extractors/integration.d.ts.map +1 -1
- package/dist/extractors/live-org/extractors/integration.js +14 -5
- package/dist/extractors/live-org/extractors/integration.js.map +1 -1
- package/dist/extractors/live-org/extractors/lwc.d.ts +2 -1
- package/dist/extractors/live-org/extractors/lwc.d.ts.map +1 -1
- package/dist/extractors/live-org/extractors/lwc.js +95 -47
- package/dist/extractors/live-org/extractors/lwc.js.map +1 -1
- package/dist/extractors/live-org/extractors/mcd-baseline.d.ts.map +1 -1
- package/dist/extractors/live-org/extractors/mcd-baseline.js +2 -4
- package/dist/extractors/live-org/extractors/mcd-baseline.js.map +1 -1
- package/dist/extractors/live-org/extractors/object.d.ts +2 -11
- package/dist/extractors/live-org/extractors/object.d.ts.map +1 -1
- package/dist/extractors/live-org/extractors/object.js +127 -226
- package/dist/extractors/live-org/extractors/object.js.map +1 -1
- package/dist/extractors/live-org/extractors/omnistudio-retrieve.d.ts.map +1 -1
- package/dist/extractors/live-org/extractors/omnistudio-retrieve.js.map +1 -1
- package/dist/extractors/live-org/extractors/omnistudio.d.ts +11 -1
- package/dist/extractors/live-org/extractors/omnistudio.d.ts.map +1 -1
- package/dist/extractors/live-org/extractors/omnistudio.js +104 -16
- package/dist/extractors/live-org/extractors/omnistudio.js.map +1 -1
- package/dist/extractors/live-org/extractors/security.d.ts +12 -1
- package/dist/extractors/live-org/extractors/security.d.ts.map +1 -1
- package/dist/extractors/live-org/extractors/security.js +29 -7
- package/dist/extractors/live-org/extractors/security.js.map +1 -1
- package/dist/extractors/live-org/rate-limit.d.ts +48 -16
- package/dist/extractors/live-org/rate-limit.d.ts.map +1 -1
- package/dist/extractors/live-org/rate-limit.js +235 -40
- package/dist/extractors/live-org/rate-limit.js.map +1 -1
- package/dist/extractors/live-org/retry.d.ts +136 -0
- package/dist/extractors/live-org/retry.d.ts.map +1 -0
- package/dist/extractors/live-org/retry.js +416 -0
- package/dist/extractors/live-org/retry.js.map +1 -0
- package/dist/extractors/live-org/vlocity/runner.d.ts +2 -1
- package/dist/extractors/live-org/vlocity/runner.d.ts.map +1 -1
- package/dist/extractors/live-org/vlocity/runner.js +124 -72
- package/dist/extractors/live-org/vlocity/runner.js.map +1 -1
- package/dist/extractors/vlocity-shared/row.d.ts +19 -0
- package/dist/extractors/vlocity-shared/row.d.ts.map +1 -0
- package/dist/extractors/vlocity-shared/row.js +69 -0
- package/dist/extractors/vlocity-shared/row.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/ingest/delta.d.ts +70 -0
- package/dist/ingest/delta.d.ts.map +1 -0
- package/dist/ingest/delta.js +534 -0
- package/dist/ingest/delta.js.map +1 -0
- package/dist/ingest/detect-deletions-guard.d.ts +79 -0
- package/dist/ingest/detect-deletions-guard.d.ts.map +1 -0
- package/dist/ingest/detect-deletions-guard.js +122 -0
- package/dist/ingest/detect-deletions-guard.js.map +1 -0
- package/dist/ingest/index.d.ts +8 -1
- package/dist/ingest/index.d.ts.map +1 -1
- package/dist/ingest/index.js +4 -0
- package/dist/ingest/index.js.map +1 -1
- package/dist/ingest/live-ingest.d.ts +180 -0
- package/dist/ingest/live-ingest.d.ts.map +1 -1
- package/dist/ingest/live-ingest.js +1617 -625
- package/dist/ingest/live-ingest.js.map +1 -1
- package/dist/ingest/lock.d.ts +47 -0
- package/dist/ingest/lock.d.ts.map +1 -0
- package/dist/ingest/lock.js +222 -0
- package/dist/ingest/lock.js.map +1 -0
- package/dist/ingest/member-rows.d.ts +88 -0
- package/dist/ingest/member-rows.d.ts.map +1 -0
- package/dist/ingest/member-rows.js +152 -0
- package/dist/ingest/member-rows.js.map +1 -0
- package/dist/ingest/offline-org.d.ts +8 -0
- package/dist/ingest/offline-org.d.ts.map +1 -0
- package/dist/ingest/offline-org.js +41 -0
- package/dist/ingest/offline-org.js.map +1 -0
- package/dist/ingest/parse-member.d.ts +53 -0
- package/dist/ingest/parse-member.d.ts.map +1 -0
- package/dist/ingest/parse-member.js +276 -0
- package/dist/ingest/parse-member.js.map +1 -0
- package/dist/ingest/parse-pool.d.ts +139 -0
- package/dist/ingest/parse-pool.d.ts.map +1 -0
- package/dist/ingest/parse-pool.js +593 -0
- package/dist/ingest/parse-pool.js.map +1 -0
- package/dist/ingest/parse-worker.d.ts +3 -0
- package/dist/ingest/parse-worker.d.ts.map +1 -0
- package/dist/ingest/parse-worker.js +100 -0
- package/dist/ingest/parse-worker.js.map +1 -0
- package/dist/ingest/profile.d.ts +27 -0
- package/dist/ingest/profile.d.ts.map +1 -0
- package/dist/ingest/profile.js +48 -0
- package/dist/ingest/profile.js.map +1 -0
- package/dist/ingest/resolver-gates.d.ts +12 -0
- package/dist/ingest/resolver-gates.d.ts.map +1 -0
- package/dist/ingest/resolver-gates.js +53 -0
- package/dist/ingest/resolver-gates.js.map +1 -0
- package/dist/parsers/apex/arity-resolver.d.ts +16 -3
- package/dist/parsers/apex/arity-resolver.d.ts.map +1 -1
- package/dist/parsers/apex/arity-resolver.js +31 -2
- package/dist/parsers/apex/arity-resolver.js.map +1 -1
- package/dist/parsers/apex/ast-extractor.d.ts +4 -0
- package/dist/parsers/apex/ast-extractor.d.ts.map +1 -1
- package/dist/parsers/apex/ast-extractor.js +128 -29
- package/dist/parsers/apex/ast-extractor.js.map +1 -1
- package/dist/parsers/apex/class-refs.d.ts +78 -0
- package/dist/parsers/apex/class-refs.d.ts.map +1 -0
- package/dist/parsers/apex/class-refs.js +465 -0
- package/dist/parsers/apex/class-refs.js.map +1 -0
- package/dist/parsers/apex/class.d.ts +158 -0
- package/dist/parsers/apex/class.d.ts.map +1 -1
- package/dist/parsers/apex/class.js +1073 -172
- package/dist/parsers/apex/class.js.map +1 -1
- package/dist/parsers/apex/dynamic-soql.d.ts +52 -0
- package/dist/parsers/apex/dynamic-soql.d.ts.map +1 -0
- package/dist/parsers/apex/dynamic-soql.js +427 -0
- package/dist/parsers/apex/dynamic-soql.js.map +1 -0
- package/dist/parsers/apex/field-access.d.ts +65 -0
- package/dist/parsers/apex/field-access.d.ts.map +1 -0
- package/dist/parsers/apex/field-access.js +438 -0
- package/dist/parsers/apex/field-access.js.map +1 -0
- package/dist/parsers/apex/interface-ref-resolver.d.ts +32 -0
- package/dist/parsers/apex/interface-ref-resolver.d.ts.map +1 -0
- package/dist/parsers/apex/interface-ref-resolver.js +64 -0
- package/dist/parsers/apex/interface-ref-resolver.js.map +1 -0
- package/dist/parsers/apex/name-refs.d.ts +83 -0
- package/dist/parsers/apex/name-refs.d.ts.map +1 -0
- package/dist/parsers/apex/name-refs.js +465 -0
- package/dist/parsers/apex/name-refs.js.map +1 -0
- package/dist/parsers/apex/page-refs.d.ts +11 -0
- package/dist/parsers/apex/page-refs.d.ts.map +1 -0
- package/dist/parsers/apex/page-refs.js +34 -0
- package/dist/parsers/apex/page-refs.js.map +1 -0
- package/dist/parsers/apex/ref-resolver.d.ts +58 -0
- package/dist/parsers/apex/ref-resolver.d.ts.map +1 -0
- package/dist/parsers/apex/ref-resolver.js +348 -0
- package/dist/parsers/apex/ref-resolver.js.map +1 -0
- package/dist/parsers/apex/soql-fields.d.ts +75 -0
- package/dist/parsers/apex/soql-fields.d.ts.map +1 -0
- package/dist/parsers/apex/soql-fields.js +338 -0
- package/dist/parsers/apex/soql-fields.js.map +1 -0
- package/dist/parsers/apex/trigger.d.ts.map +1 -1
- package/dist/parsers/apex/trigger.js +100 -2
- package/dist/parsers/apex/trigger.js.map +1 -1
- package/dist/parsers/apex/vlocity-calls.d.ts +25 -0
- package/dist/parsers/apex/vlocity-calls.d.ts.map +1 -0
- package/dist/parsers/apex/vlocity-calls.js +225 -0
- package/dist/parsers/apex/vlocity-calls.js.map +1 -0
- package/dist/parsers/aura/bundle.d.ts +39 -0
- package/dist/parsers/aura/bundle.d.ts.map +1 -0
- package/dist/parsers/aura/bundle.js +269 -0
- package/dist/parsers/aura/bundle.js.map +1 -0
- package/dist/parsers/aura/embed-resolver.d.ts +17 -0
- package/dist/parsers/aura/embed-resolver.d.ts.map +1 -0
- package/dist/parsers/aura/embed-resolver.js +58 -0
- package/dist/parsers/aura/embed-resolver.js.map +1 -0
- package/dist/parsers/aura/index.d.ts +4 -0
- package/dist/parsers/aura/index.d.ts.map +1 -0
- package/dist/parsers/aura/index.js +6 -0
- package/dist/parsers/aura/index.js.map +1 -0
- package/dist/parsers/common.d.ts +9 -0
- package/dist/parsers/common.d.ts.map +1 -1
- package/dist/parsers/common.js +17 -0
- package/dist/parsers/common.js.map +1 -1
- package/dist/parsers/cross-flavor-resolver.js +8 -2
- package/dist/parsers/cross-flavor-resolver.js.map +1 -1
- package/dist/parsers/flow/invocable-resolver.d.ts.map +1 -1
- package/dist/parsers/flow/invocable-resolver.js +2 -0
- package/dist/parsers/flow/invocable-resolver.js.map +1 -1
- package/dist/parsers/flow/parser.d.ts +9 -0
- package/dist/parsers/flow/parser.d.ts.map +1 -1
- package/dist/parsers/flow/parser.js +327 -2
- package/dist/parsers/flow/parser.js.map +1 -1
- package/dist/parsers/flow/relationship-resolver.d.ts +34 -0
- package/dist/parsers/flow/relationship-resolver.d.ts.map +1 -0
- package/dist/parsers/flow/relationship-resolver.js +199 -0
- package/dist/parsers/flow/relationship-resolver.js.map +1 -0
- package/dist/parsers/generic/opaque-node-parser.d.ts +5 -1
- package/dist/parsers/generic/opaque-node-parser.d.ts.map +1 -1
- package/dist/parsers/generic/opaque-node-parser.js +37 -5
- package/dist/parsers/generic/opaque-node-parser.js.map +1 -1
- package/dist/parsers/generic/reflection-walker.d.ts +7 -0
- package/dist/parsers/generic/reflection-walker.d.ts.map +1 -1
- package/dist/parsers/generic/reflection-walker.js +38 -13
- package/dist/parsers/generic/reflection-walker.js.map +1 -1
- package/dist/parsers/hash-salts.d.ts +72 -0
- package/dist/parsers/hash-salts.d.ts.map +1 -0
- package/dist/parsers/hash-salts.js +91 -0
- package/dist/parsers/hash-salts.js.map +1 -0
- package/dist/parsers/index.d.ts +2 -0
- package/dist/parsers/index.d.ts.map +1 -1
- package/dist/parsers/index.js +2 -0
- package/dist/parsers/index.js.map +1 -1
- package/dist/parsers/lwc/bundle.d.ts +1 -0
- package/dist/parsers/lwc/bundle.d.ts.map +1 -1
- package/dist/parsers/lwc/bundle.js +37 -2
- package/dist/parsers/lwc/bundle.js.map +1 -1
- package/dist/parsers/lwc/field-key-resolver.d.ts +32 -0
- package/dist/parsers/lwc/field-key-resolver.d.ts.map +1 -0
- package/dist/parsers/lwc/field-key-resolver.js +92 -0
- package/dist/parsers/lwc/field-key-resolver.js.map +1 -0
- package/dist/parsers/lwc/html-visitor.d.ts +1 -1
- package/dist/parsers/lwc/html-visitor.d.ts.map +1 -1
- package/dist/parsers/lwc/html-visitor.js +81 -3
- package/dist/parsers/lwc/html-visitor.js.map +1 -1
- package/dist/parsers/lwc/js-visitor.d.ts +16 -0
- package/dist/parsers/lwc/js-visitor.d.ts.map +1 -1
- package/dist/parsers/lwc/js-visitor.js +354 -12
- package/dist/parsers/lwc/js-visitor.js.map +1 -1
- package/dist/parsers/lwc/omnidef.d.ts +10 -0
- package/dist/parsers/lwc/omnidef.d.ts.map +1 -0
- package/dist/parsers/lwc/omnidef.js +127 -0
- package/dist/parsers/lwc/omnidef.js.map +1 -0
- package/dist/parsers/lwc/xml-meta.d.ts.map +1 -1
- package/dist/parsers/lwc/xml-meta.js +16 -1
- package/dist/parsers/lwc/xml-meta.js.map +1 -1
- package/dist/parsers/object/formula-hop-resolver.d.ts +33 -0
- package/dist/parsers/object/formula-hop-resolver.d.ts.map +1 -0
- package/dist/parsers/object/formula-hop-resolver.js +60 -0
- package/dist/parsers/object/formula-hop-resolver.js.map +1 -0
- package/dist/parsers/object/formula-refs.d.ts +40 -0
- package/dist/parsers/object/formula-refs.d.ts.map +1 -0
- package/dist/parsers/object/formula-refs.js +129 -0
- package/dist/parsers/object/formula-refs.js.map +1 -0
- package/dist/parsers/object/object.d.ts +3 -0
- package/dist/parsers/object/object.d.ts.map +1 -1
- package/dist/parsers/object/object.js +54 -57
- package/dist/parsers/object/object.js.map +1 -1
- package/dist/parsers/omnistudio/data-transform.d.ts.map +1 -1
- package/dist/parsers/omnistudio/data-transform.js +40 -7
- package/dist/parsers/omnistudio/data-transform.js.map +1 -1
- package/dist/parsers/omnistudio/integration-procedure.d.ts.map +1 -1
- package/dist/parsers/omnistudio/integration-procedure.js +6 -6
- package/dist/parsers/omnistudio/integration-procedure.js.map +1 -1
- package/dist/parsers/omnistudio/overlap-detector.d.ts.map +1 -1
- package/dist/parsers/omnistudio/overlap-detector.js +29 -11
- package/dist/parsers/omnistudio/overlap-detector.js.map +1 -1
- package/dist/parsers/omnistudio/process.d.ts.map +1 -1
- package/dist/parsers/omnistudio/process.js +6 -6
- package/dist/parsers/omnistudio/process.js.map +1 -1
- package/dist/parsers/omnistudio/remote-refs.d.ts +11 -0
- package/dist/parsers/omnistudio/remote-refs.d.ts.map +1 -0
- package/dist/parsers/omnistudio/remote-refs.js +17 -0
- package/dist/parsers/omnistudio/remote-refs.js.map +1 -0
- package/dist/parsers/omnistudio/ui-card.d.ts.map +1 -1
- package/dist/parsers/omnistudio/ui-card.js +6 -6
- package/dist/parsers/omnistudio/ui-card.js.map +1 -1
- package/dist/parsers/resolver-prune.d.ts +14 -0
- package/dist/parsers/resolver-prune.d.ts.map +1 -0
- package/dist/parsers/resolver-prune.js +29 -0
- package/dist/parsers/resolver-prune.js.map +1 -0
- package/dist/parsers/rules/_engine.d.ts.map +1 -1
- package/dist/parsers/rules/_engine.js +4 -0
- package/dist/parsers/rules/_engine.js.map +1 -1
- package/dist/parsers/rules/_engine.ts +3 -0
- package/dist/parsers/rules/_loader.d.ts +8 -0
- package/dist/parsers/rules/_loader.d.ts.map +1 -1
- package/dist/parsers/rules/_loader.js +17 -0
- package/dist/parsers/rules/_loader.js.map +1 -1
- package/dist/parsers/rules/_loader.ts +19 -0
- package/dist/parsers/rules/_schema.d.ts +12 -12
- package/dist/parsers/rules/_selectors.d.ts +1 -0
- package/dist/parsers/rules/_selectors.d.ts.map +1 -1
- package/dist/parsers/rules/_selectors.js +6 -0
- package/dist/parsers/rules/_selectors.js.map +1 -1
- package/dist/parsers/rules/_selectors.ts +6 -0
- package/dist/parsers/rules/apex-component.yml +49 -4
- package/dist/parsers/rules/apex-page.yml +49 -4
- package/dist/parsers/rules/approval-process.yml +25 -5
- package/dist/parsers/rules/custom-metadata-type.yml +14 -7
- package/dist/parsers/rules/custom-setting.yml +24 -0
- package/dist/parsers/rules/custom-tab.yml +41 -0
- package/dist/parsers/rules/lightning-page.yml +4 -2
- package/dist/parsers/rules/workflow.yml +30 -10
- package/dist/parsers/vf/refs.d.ts +39 -0
- package/dist/parsers/vf/refs.d.ts.map +1 -0
- package/dist/parsers/vf/refs.js +152 -0
- package/dist/parsers/vf/refs.js.map +1 -0
- package/dist/parsers/vlocity/apex-resolver.d.ts +21 -0
- package/dist/parsers/vlocity/apex-resolver.d.ts.map +1 -0
- package/dist/parsers/vlocity/apex-resolver.js +115 -0
- package/dist/parsers/vlocity/apex-resolver.js.map +1 -0
- package/dist/parsers/vlocity/common.d.ts +72 -1
- package/dist/parsers/vlocity/common.d.ts.map +1 -1
- package/dist/parsers/vlocity/common.js +332 -2
- package/dist/parsers/vlocity/common.js.map +1 -1
- package/dist/parsers/vlocity/data-raptor.d.ts +9 -0
- package/dist/parsers/vlocity/data-raptor.d.ts.map +1 -1
- package/dist/parsers/vlocity/data-raptor.js +148 -17
- package/dist/parsers/vlocity/data-raptor.js.map +1 -1
- package/dist/parsers/vlocity/datapack-refs.d.ts +22 -0
- package/dist/parsers/vlocity/datapack-refs.d.ts.map +1 -0
- package/dist/parsers/vlocity/datapack-refs.js +126 -0
- package/dist/parsers/vlocity/datapack-refs.js.map +1 -0
- package/dist/parsers/vlocity/dr-relationship-resolver.d.ts +26 -0
- package/dist/parsers/vlocity/dr-relationship-resolver.d.ts.map +1 -0
- package/dist/parsers/vlocity/dr-relationship-resolver.js +137 -0
- package/dist/parsers/vlocity/dr-relationship-resolver.js.map +1 -0
- package/dist/parsers/vlocity/integration-procedure.d.ts.map +1 -1
- package/dist/parsers/vlocity/integration-procedure.js +15 -48
- package/dist/parsers/vlocity/integration-procedure.js.map +1 -1
- package/dist/parsers/vlocity/name-match-resolver.d.ts +21 -0
- package/dist/parsers/vlocity/name-match-resolver.d.ts.map +1 -0
- package/dist/parsers/vlocity/name-match-resolver.js +97 -0
- package/dist/parsers/vlocity/name-match-resolver.js.map +1 -0
- package/dist/parsers/vlocity/omni-script.d.ts.map +1 -1
- package/dist/parsers/vlocity/omni-script.js +18 -32
- package/dist/parsers/vlocity/omni-script.js.map +1 -1
- package/dist/parsers/vlocity/vlocity-card.d.ts.map +1 -1
- package/dist/parsers/vlocity/vlocity-card.js +30 -36
- package/dist/parsers/vlocity/vlocity-card.js.map +1 -1
- package/dist/render/mermaid/dependency-graph.d.ts.map +1 -1
- package/dist/render/mermaid/dependency-graph.js +13 -1
- package/dist/render/mermaid/dependency-graph.js.map +1 -1
- package/dist/render/mermaid/diff.d.ts.map +1 -1
- package/dist/render/mermaid/diff.js +4 -2
- package/dist/render/mermaid/diff.js.map +1 -1
- package/dist/render/mermaid/shared.d.ts +7 -0
- package/dist/render/mermaid/shared.d.ts.map +1 -1
- package/dist/render/mermaid/shared.js +15 -6
- package/dist/render/mermaid/shared.js.map +1 -1
- package/dist/render/sarif.d.ts.map +1 -1
- package/dist/render/sarif.js +1 -3
- package/dist/render/sarif.js.map +1 -1
- package/dist/storage/index.d.ts +3 -3
- package/dist/storage/index.d.ts.map +1 -1
- package/dist/storage/index.js +2 -2
- package/dist/storage/index.js.map +1 -1
- package/dist/storage/interfaces.d.ts +212 -1
- package/dist/storage/interfaces.d.ts.map +1 -1
- package/dist/storage/sqlite/graph-store.d.ts +366 -3
- package/dist/storage/sqlite/graph-store.d.ts.map +1 -1
- package/dist/storage/sqlite/graph-store.js +1755 -68
- package/dist/storage/sqlite/graph-store.js.map +1 -1
- package/dist/storage/sqlite/migrations.d.ts +11 -0
- package/dist/storage/sqlite/migrations.d.ts.map +1 -1
- package/dist/storage/sqlite/migrations.js +245 -11
- package/dist/storage/sqlite/migrations.js.map +1 -1
- package/dist/storage/sqlite/rename-stability.d.ts +1 -1
- package/dist/storage/sqlite/rename-stability.d.ts.map +1 -1
- package/dist/storage/sqlite/rename-stability.js +2 -6
- package/dist/storage/sqlite/rename-stability.js.map +1 -1
- package/dist/storage/sqlite/snapshot-store.d.ts +79 -3
- package/dist/storage/sqlite/snapshot-store.d.ts.map +1 -1
- package/dist/storage/sqlite/snapshot-store.js +500 -100
- package/dist/storage/sqlite/snapshot-store.js.map +1 -1
- package/dist/storage/sqlite/snapshot-versions.d.ts +51 -0
- package/dist/storage/sqlite/snapshot-versions.d.ts.map +1 -0
- package/dist/storage/sqlite/snapshot-versions.js +209 -0
- package/dist/storage/sqlite/snapshot-versions.js.map +1 -0
- package/dist/storage/sqlite/vector-store.d.ts +12 -0
- package/dist/storage/sqlite/vector-store.d.ts.map +1 -1
- package/dist/storage/sqlite/vector-store.js +57 -14
- package/dist/storage/sqlite/vector-store.js.map +1 -1
- package/dist/util/process-owner.d.ts +35 -0
- package/dist/util/process-owner.d.ts.map +1 -0
- package/dist/util/process-owner.js +87 -0
- package/dist/util/process-owner.js.map +1 -0
- package/dist/util/worker-heap-cap.d.ts +10 -0
- package/dist/util/worker-heap-cap.d.ts.map +1 -0
- package/dist/util/worker-heap-cap.js +65 -0
- package/dist/util/worker-heap-cap.js.map +1 -0
- package/package.json +6 -7
- package/dist/analyze/__tests__/audit-graph.test.d.ts +0 -2
- package/dist/analyze/__tests__/audit-graph.test.d.ts.map +0 -1
- package/dist/analyze/__tests__/audit-graph.test.js +0 -122
- package/dist/analyze/__tests__/audit-graph.test.js.map +0 -1
- package/dist/analyze/__tests__/cross-org-two-store.test.d.ts +0 -2
- package/dist/analyze/__tests__/cross-org-two-store.test.d.ts.map +0 -1
- package/dist/analyze/__tests__/cross-org-two-store.test.js +0 -68
- package/dist/analyze/__tests__/cross-org-two-store.test.js.map +0 -1
- package/dist/analyze/__tests__/dead-code-confidence.test.d.ts +0 -2
- package/dist/analyze/__tests__/dead-code-confidence.test.d.ts.map +0 -1
- package/dist/analyze/__tests__/dead-code-confidence.test.js +0 -66
- package/dist/analyze/__tests__/dead-code-confidence.test.js.map +0 -1
- package/dist/analyze/__tests__/deployment-manifest.test.d.ts +0 -2
- package/dist/analyze/__tests__/deployment-manifest.test.d.ts.map +0 -1
- package/dist/analyze/__tests__/deployment-manifest.test.js +0 -66
- package/dist/analyze/__tests__/deployment-manifest.test.js.map +0 -1
- package/dist/analyze/__tests__/find-nodes.test.d.ts +0 -2
- package/dist/analyze/__tests__/find-nodes.test.d.ts.map +0 -1
- package/dist/analyze/__tests__/find-nodes.test.js +0 -129
- package/dist/analyze/__tests__/find-nodes.test.js.map +0 -1
- package/dist/analyze/__tests__/findings.test.d.ts +0 -2
- package/dist/analyze/__tests__/findings.test.d.ts.map +0 -1
- package/dist/analyze/__tests__/findings.test.js +0 -90
- package/dist/analyze/__tests__/findings.test.js.map +0 -1
- package/dist/analyze/__tests__/governor-risks-detection.test.d.ts +0 -2
- package/dist/analyze/__tests__/governor-risks-detection.test.d.ts.map +0 -1
- package/dist/analyze/__tests__/governor-risks-detection.test.js +0 -70
- package/dist/analyze/__tests__/governor-risks-detection.test.js.map +0 -1
- package/dist/analyze/__tests__/populate.test.d.ts +0 -2
- package/dist/analyze/__tests__/populate.test.d.ts.map +0 -1
- package/dist/analyze/__tests__/populate.test.js +0 -129
- package/dist/analyze/__tests__/populate.test.js.map +0 -1
- package/dist/analyze/__tests__/traversal-caps.test.d.ts +0 -2
- package/dist/analyze/__tests__/traversal-caps.test.d.ts.map +0 -1
- package/dist/analyze/__tests__/traversal-caps.test.js +0 -129
- package/dist/analyze/__tests__/traversal-caps.test.js.map +0 -1
- package/dist/analyze/__tests__/wip-impact.test.d.ts +0 -2
- package/dist/analyze/__tests__/wip-impact.test.d.ts.map +0 -1
- package/dist/analyze/__tests__/wip-impact.test.js +0 -134
- package/dist/analyze/__tests__/wip-impact.test.js.map +0 -1
- package/dist/embedding/__tests__/embed.test.d.ts +0 -2
- package/dist/embedding/__tests__/embed.test.d.ts.map +0 -1
- package/dist/embedding/__tests__/embed.test.js +0 -72
- package/dist/embedding/__tests__/embed.test.js.map +0 -1
- package/dist/embedding/__tests__/queue.test.d.ts +0 -2
- package/dist/embedding/__tests__/queue.test.d.ts.map +0 -1
- package/dist/embedding/__tests__/queue.test.js +0 -64
- package/dist/embedding/__tests__/queue.test.js.map +0 -1
- package/dist/extractors/__tests__/network-egress.test.d.ts +0 -2
- package/dist/extractors/__tests__/network-egress.test.d.ts.map +0 -1
- package/dist/extractors/__tests__/network-egress.test.js +0 -32
- package/dist/extractors/__tests__/network-egress.test.js.map +0 -1
- package/dist/extractors/filesystem/__tests__/filesystem.test.d.ts +0 -2
- package/dist/extractors/filesystem/__tests__/filesystem.test.d.ts.map +0 -1
- package/dist/extractors/filesystem/__tests__/filesystem.test.js +0 -96
- package/dist/extractors/filesystem/__tests__/filesystem.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/_jsforce-mock.d.ts +0 -38
- package/dist/extractors/live-org/__tests__/_jsforce-mock.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/_jsforce-mock.js +0 -96
- package/dist/extractors/live-org/__tests__/_jsforce-mock.js.map +0 -1
- package/dist/extractors/live-org/__tests__/auth.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/auth.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/auth.test.js +0 -85
- package/dist/extractors/live-org/__tests__/auth.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/bulk-retrieve.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/bulk-retrieve.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/bulk-retrieve.test.js +0 -150
- package/dist/extractors/live-org/__tests__/bulk-retrieve.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/capabilities.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/capabilities.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/capabilities.test.js +0 -54
- package/dist/extractors/live-org/__tests__/capabilities.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/discovery.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/discovery.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/discovery.test.js +0 -55
- package/dist/extractors/live-org/__tests__/discovery.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/dispatch.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/dispatch.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/dispatch.test.js +0 -68
- package/dist/extractors/live-org/__tests__/dispatch.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/extractor-enrichment.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/extractor-enrichment.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/extractor-enrichment.test.js +0 -312
- package/dist/extractors/live-org/__tests__/extractor-enrichment.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/liveness.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/liveness.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/liveness.test.js +0 -148
- package/dist/extractors/live-org/__tests__/liveness.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/mcd-baseline.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/mcd-baseline.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/mcd-baseline.test.js +0 -174
- package/dist/extractors/live-org/__tests__/mcd-baseline.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/metadata-read-batch.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/metadata-read-batch.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/metadata-read-batch.test.js +0 -75
- package/dist/extractors/live-org/__tests__/metadata-read-batch.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/omnistudio-retrieve.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/omnistudio-retrieve.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/omnistudio-retrieve.test.js +0 -135
- package/dist/extractors/live-org/__tests__/omnistudio-retrieve.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/pools.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/pools.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/pools.test.js +0 -75
- package/dist/extractors/live-org/__tests__/pools.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/rate-limit.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/rate-limit.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/rate-limit.test.js +0 -78
- package/dist/extractors/live-org/__tests__/rate-limit.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/read-only-proxy.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/read-only-proxy.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/read-only-proxy.test.js +0 -376
- package/dist/extractors/live-org/__tests__/read-only-proxy.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/soql-rebatch.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/soql-rebatch.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/soql-rebatch.test.js +0 -118
- package/dist/extractors/live-org/__tests__/soql-rebatch.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/source-member.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/source-member.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/source-member.test.js +0 -84
- package/dist/extractors/live-org/__tests__/source-member.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/vlocity-runner.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/vlocity-runner.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/vlocity-runner.test.js +0 -115
- package/dist/extractors/live-org/__tests__/vlocity-runner.test.js.map +0 -1
- package/dist/ingest/__tests__/full-sync-deletion.test.d.ts +0 -2
- package/dist/ingest/__tests__/full-sync-deletion.test.d.ts.map +0 -1
- package/dist/ingest/__tests__/full-sync-deletion.test.js +0 -200
- package/dist/ingest/__tests__/full-sync-deletion.test.js.map +0 -1
- package/dist/ingest/__tests__/incremental-deletion.test.d.ts +0 -2
- package/dist/ingest/__tests__/incremental-deletion.test.d.ts.map +0 -1
- package/dist/ingest/__tests__/incremental-deletion.test.js +0 -92
- package/dist/ingest/__tests__/incremental-deletion.test.js.map +0 -1
- package/dist/ingest/__tests__/live-ingest.test.d.ts +0 -2
- package/dist/ingest/__tests__/live-ingest.test.d.ts.map +0 -1
- package/dist/ingest/__tests__/live-ingest.test.js +0 -155
- package/dist/ingest/__tests__/live-ingest.test.js.map +0 -1
- package/dist/ingest/__tests__/multi-org-orchestrator.test.d.ts +0 -2
- package/dist/ingest/__tests__/multi-org-orchestrator.test.d.ts.map +0 -1
- package/dist/ingest/__tests__/multi-org-orchestrator.test.js +0 -108
- package/dist/ingest/__tests__/multi-org-orchestrator.test.js.map +0 -1
- package/dist/ingest/__tests__/read-only-wrapping.test.d.ts +0 -2
- package/dist/ingest/__tests__/read-only-wrapping.test.d.ts.map +0 -1
- package/dist/ingest/__tests__/read-only-wrapping.test.js +0 -62
- package/dist/ingest/__tests__/read-only-wrapping.test.js.map +0 -1
- package/dist/parsers/__tests__/_harness.d.ts +0 -13
- package/dist/parsers/__tests__/_harness.d.ts.map +0 -1
- package/dist/parsers/__tests__/_harness.js +0 -84
- package/dist/parsers/__tests__/_harness.js.map +0 -1
- package/dist/parsers/__tests__/apex.golden.test.d.ts +0 -2
- package/dist/parsers/__tests__/apex.golden.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/apex.golden.test.js +0 -28
- package/dist/parsers/__tests__/apex.golden.test.js.map +0 -1
- package/dist/parsers/__tests__/cross-flavor.test.d.ts +0 -2
- package/dist/parsers/__tests__/cross-flavor.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/cross-flavor.test.js +0 -56
- package/dist/parsers/__tests__/cross-flavor.test.js.map +0 -1
- package/dist/parsers/__tests__/edge-provenance.test.d.ts +0 -2
- package/dist/parsers/__tests__/edge-provenance.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/edge-provenance.test.js +0 -44
- package/dist/parsers/__tests__/edge-provenance.test.js.map +0 -1
- package/dist/parsers/__tests__/flow.golden.test.d.ts +0 -2
- package/dist/parsers/__tests__/flow.golden.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/flow.golden.test.js +0 -17
- package/dist/parsers/__tests__/flow.golden.test.js.map +0 -1
- package/dist/parsers/__tests__/integration.golden.test.d.ts +0 -2
- package/dist/parsers/__tests__/integration.golden.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/integration.golden.test.js +0 -31
- package/dist/parsers/__tests__/integration.golden.test.js.map +0 -1
- package/dist/parsers/__tests__/long-tail.golden.test.d.ts +0 -2
- package/dist/parsers/__tests__/long-tail.golden.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/long-tail.golden.test.js +0 -115
- package/dist/parsers/__tests__/long-tail.golden.test.js.map +0 -1
- package/dist/parsers/__tests__/lwc-bindings.test.d.ts +0 -2
- package/dist/parsers/__tests__/lwc-bindings.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/lwc-bindings.test.js +0 -146
- package/dist/parsers/__tests__/lwc-bindings.test.js.map +0 -1
- package/dist/parsers/__tests__/lwc.golden.test.d.ts +0 -2
- package/dist/parsers/__tests__/lwc.golden.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/lwc.golden.test.js +0 -16
- package/dist/parsers/__tests__/lwc.golden.test.js.map +0 -1
- package/dist/parsers/__tests__/mcd-gap-fills.test.d.ts +0 -2
- package/dist/parsers/__tests__/mcd-gap-fills.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/mcd-gap-fills.test.js +0 -75
- package/dist/parsers/__tests__/mcd-gap-fills.test.js.map +0 -1
- package/dist/parsers/__tests__/object-inline-fields.test.d.ts +0 -2
- package/dist/parsers/__tests__/object-inline-fields.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/object-inline-fields.test.js +0 -115
- package/dist/parsers/__tests__/object-inline-fields.test.js.map +0 -1
- package/dist/parsers/__tests__/object.golden.test.d.ts +0 -2
- package/dist/parsers/__tests__/object.golden.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/object.golden.test.js +0 -33
- package/dist/parsers/__tests__/object.golden.test.js.map +0 -1
- package/dist/parsers/__tests__/omnistudio-overlap.test.d.ts +0 -2
- package/dist/parsers/__tests__/omnistudio-overlap.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/omnistudio-overlap.test.js +0 -163
- package/dist/parsers/__tests__/omnistudio-overlap.test.js.map +0 -1
- package/dist/parsers/__tests__/omnistudio.golden.test.d.ts +0 -2
- package/dist/parsers/__tests__/omnistudio.golden.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/omnistudio.golden.test.js +0 -54
- package/dist/parsers/__tests__/omnistudio.golden.test.js.map +0 -1
- package/dist/parsers/__tests__/reflection-walker.test.d.ts +0 -2
- package/dist/parsers/__tests__/reflection-walker.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/reflection-walker.test.js +0 -162
- package/dist/parsers/__tests__/reflection-walker.test.js.map +0 -1
- package/dist/parsers/__tests__/registry.test.d.ts +0 -2
- package/dist/parsers/__tests__/registry.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/registry.test.js +0 -23
- package/dist/parsers/__tests__/registry.test.js.map +0 -1
- package/dist/parsers/__tests__/security.golden.test.d.ts +0 -2
- package/dist/parsers/__tests__/security.golden.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/security.golden.test.js +0 -31
- package/dist/parsers/__tests__/security.golden.test.js.map +0 -1
- package/dist/parsers/__tests__/vlocity.golden.test.d.ts +0 -2
- package/dist/parsers/__tests__/vlocity.golden.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/vlocity.golden.test.js +0 -109
- package/dist/parsers/__tests__/vlocity.golden.test.js.map +0 -1
- package/dist/parsers/__tests__/worker-pool.test.d.ts +0 -2
- package/dist/parsers/__tests__/worker-pool.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/worker-pool.test.js +0 -48
- package/dist/parsers/__tests__/worker-pool.test.js.map +0 -1
- package/dist/parsers/apex/__tests__/arity-resolver.test.d.ts +0 -2
- package/dist/parsers/apex/__tests__/arity-resolver.test.d.ts.map +0 -1
- package/dist/parsers/apex/__tests__/arity-resolver.test.js +0 -135
- package/dist/parsers/apex/__tests__/arity-resolver.test.js.map +0 -1
- package/dist/parsers/apex/__tests__/ast-extractor.test.d.ts +0 -2
- package/dist/parsers/apex/__tests__/ast-extractor.test.d.ts.map +0 -1
- package/dist/parsers/apex/__tests__/ast-extractor.test.js +0 -146
- package/dist/parsers/apex/__tests__/ast-extractor.test.js.map +0 -1
- package/dist/parsers/apex/__tests__/is-test-attribute.test.d.ts +0 -2
- package/dist/parsers/apex/__tests__/is-test-attribute.test.d.ts.map +0 -1
- package/dist/parsers/apex/__tests__/is-test-attribute.test.js +0 -54
- package/dist/parsers/apex/__tests__/is-test-attribute.test.js.map +0 -1
- package/dist/parsers/apex/__tests__/regex-arg-count.test.d.ts +0 -2
- package/dist/parsers/apex/__tests__/regex-arg-count.test.d.ts.map +0 -1
- package/dist/parsers/apex/__tests__/regex-arg-count.test.js +0 -82
- package/dist/parsers/apex/__tests__/regex-arg-count.test.js.map +0 -1
- package/dist/parsers/flow/__tests__/invocable-resolver.test.d.ts +0 -2
- package/dist/parsers/flow/__tests__/invocable-resolver.test.d.ts.map +0 -1
- package/dist/parsers/flow/__tests__/invocable-resolver.test.js +0 -91
- package/dist/parsers/flow/__tests__/invocable-resolver.test.js.map +0 -1
- package/dist/parsers/rules/__tests__/applies-to.test.d.ts +0 -2
- package/dist/parsers/rules/__tests__/applies-to.test.d.ts.map +0 -1
- package/dist/parsers/rules/__tests__/applies-to.test.js +0 -31
- package/dist/parsers/rules/__tests__/applies-to.test.js.map +0 -1
- package/dist/parsers/rules/__tests__/applies-to.test.ts +0 -40
- package/dist/parsers/rules/__tests__/engine.test.d.ts +0 -2
- package/dist/parsers/rules/__tests__/engine.test.d.ts.map +0 -1
- package/dist/parsers/rules/__tests__/engine.test.js +0 -127
- package/dist/parsers/rules/__tests__/engine.test.js.map +0 -1
- package/dist/parsers/rules/__tests__/engine.test.ts +0 -138
- package/dist/parsers/rules/__tests__/loader.test.d.ts +0 -2
- package/dist/parsers/rules/__tests__/loader.test.d.ts.map +0 -1
- package/dist/parsers/rules/__tests__/loader.test.js +0 -31
- package/dist/parsers/rules/__tests__/loader.test.js.map +0 -1
- package/dist/parsers/rules/__tests__/loader.test.ts +0 -37
- package/dist/parsers/rules/__tests__/schema.test.d.ts +0 -2
- package/dist/parsers/rules/__tests__/schema.test.d.ts.map +0 -1
- package/dist/parsers/rules/__tests__/schema.test.js +0 -33
- package/dist/parsers/rules/__tests__/schema.test.js.map +0 -1
- package/dist/parsers/rules/__tests__/schema.test.ts +0 -37
- package/dist/parsers/rules/__tests__/selectors.test.d.ts +0 -2
- package/dist/parsers/rules/__tests__/selectors.test.d.ts.map +0 -1
- package/dist/parsers/rules/__tests__/selectors.test.js +0 -51
- package/dist/parsers/rules/__tests__/selectors.test.js.map +0 -1
- package/dist/parsers/rules/__tests__/selectors.test.ts +0 -62
- package/dist/render/__tests__/sarif.test.d.ts +0 -2
- package/dist/render/__tests__/sarif.test.d.ts.map +0 -1
- package/dist/render/__tests__/sarif.test.js +0 -129
- package/dist/render/__tests__/sarif.test.js.map +0 -1
- package/dist/render/mermaid/__tests__/mermaid.test.d.ts +0 -2
- package/dist/render/mermaid/__tests__/mermaid.test.d.ts.map +0 -1
- package/dist/render/mermaid/__tests__/mermaid.test.js +0 -74
- package/dist/render/mermaid/__tests__/mermaid.test.js.map +0 -1
- package/dist/storage/__tests__/analysis-tables-migration.test.d.ts +0 -2
- package/dist/storage/__tests__/analysis-tables-migration.test.d.ts.map +0 -1
- package/dist/storage/__tests__/analysis-tables-migration.test.js +0 -44
- package/dist/storage/__tests__/analysis-tables-migration.test.js.map +0 -1
- package/dist/storage/__tests__/graph-store.test.d.ts +0 -2
- package/dist/storage/__tests__/graph-store.test.d.ts.map +0 -1
- package/dist/storage/__tests__/graph-store.test.js +0 -132
- package/dist/storage/__tests__/graph-store.test.js.map +0 -1
- package/dist/storage/__tests__/identifier.test.d.ts +0 -2
- package/dist/storage/__tests__/identifier.test.d.ts.map +0 -1
- package/dist/storage/__tests__/identifier.test.js +0 -35
- package/dist/storage/__tests__/identifier.test.js.map +0 -1
- package/dist/storage/__tests__/load-better-sqlite3.test.d.ts +0 -2
- package/dist/storage/__tests__/load-better-sqlite3.test.d.ts.map +0 -1
- package/dist/storage/__tests__/load-better-sqlite3.test.js +0 -65
- package/dist/storage/__tests__/load-better-sqlite3.test.js.map +0 -1
- package/dist/storage/__tests__/migrations.test.d.ts +0 -2
- package/dist/storage/__tests__/migrations.test.d.ts.map +0 -1
- package/dist/storage/__tests__/migrations.test.js +0 -93
- package/dist/storage/__tests__/migrations.test.js.map +0 -1
- package/dist/storage/__tests__/perf-50k.test.d.ts +0 -2
- package/dist/storage/__tests__/perf-50k.test.d.ts.map +0 -1
- package/dist/storage/__tests__/perf-50k.test.js +0 -67
- package/dist/storage/__tests__/perf-50k.test.js.map +0 -1
- package/dist/storage/__tests__/rename-stability.test.d.ts +0 -2
- package/dist/storage/__tests__/rename-stability.test.d.ts.map +0 -1
- package/dist/storage/__tests__/rename-stability.test.js +0 -135
- package/dist/storage/__tests__/rename-stability.test.js.map +0 -1
- package/dist/storage/__tests__/snapshot-store.test.d.ts +0 -2
- package/dist/storage/__tests__/snapshot-store.test.d.ts.map +0 -1
- package/dist/storage/__tests__/snapshot-store.test.js +0 -117
- package/dist/storage/__tests__/snapshot-store.test.js.map +0 -1
- package/dist/storage/__tests__/snippets.test.d.ts +0 -2
- package/dist/storage/__tests__/snippets.test.d.ts.map +0 -1
- package/dist/storage/__tests__/snippets.test.js +0 -88
- package/dist/storage/__tests__/snippets.test.js.map +0 -1
- package/dist/storage/__tests__/vector-store-load-failure.test.d.ts +0 -2
- package/dist/storage/__tests__/vector-store-load-failure.test.d.ts.map +0 -1
- package/dist/storage/__tests__/vector-store-load-failure.test.js +0 -42
- package/dist/storage/__tests__/vector-store-load-failure.test.js.map +0 -1
- package/dist/storage/__tests__/vector-store.test.d.ts +0 -2
- package/dist/storage/__tests__/vector-store.test.d.ts.map +0 -1
- package/dist/storage/__tests__/vector-store.test.js +0 -87
- package/dist/storage/__tests__/vector-store.test.js.map +0 -1
- package/dist/telemetry/__tests__/event-schema.test.d.ts +0 -2
- package/dist/telemetry/__tests__/event-schema.test.d.ts.map +0 -1
- package/dist/telemetry/__tests__/event-schema.test.js +0 -56
- package/dist/telemetry/__tests__/event-schema.test.js.map +0 -1
- package/dist/telemetry/__tests__/machine-id.test.d.ts +0 -2
- package/dist/telemetry/__tests__/machine-id.test.d.ts.map +0 -1
- package/dist/telemetry/__tests__/machine-id.test.js +0 -33
- package/dist/telemetry/__tests__/machine-id.test.js.map +0 -1
- package/dist/telemetry/__tests__/pipeline.test.d.ts +0 -2
- package/dist/telemetry/__tests__/pipeline.test.d.ts.map +0 -1
- package/dist/telemetry/__tests__/pipeline.test.js +0 -68
- package/dist/telemetry/__tests__/pipeline.test.js.map +0 -1
- package/dist/telemetry/__tests__/sanitizer.test.d.ts +0 -2
- package/dist/telemetry/__tests__/sanitizer.test.d.ts.map +0 -1
- package/dist/telemetry/__tests__/sanitizer.test.js +0 -143
- package/dist/telemetry/__tests__/sanitizer.test.js.map +0 -1
|
@@ -1,10 +1,91 @@
|
|
|
1
|
-
import { mkdirSync } from "node:fs";
|
|
1
|
+
import { mkdirSync, statSync } from "node:fs";
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import { StorageError, asOrgId, asQualifiedName, asSha256 } from "@ryanstark24/sfgraph-shared";
|
|
4
4
|
import Database from "better-sqlite3";
|
|
5
|
+
import { splitIdentifierWords } from "../../embedding/tokenize.js";
|
|
6
|
+
import { isThisHost, ownStartMs, ownerIsStale, thisHost } from "../../util/process-owner.js";
|
|
5
7
|
import { edgeTableName, nodeTableName, validateLabel, validateRelType } from "../identifier.js";
|
|
6
8
|
import { wrapAbiError } from "./load-better-sqlite3.js";
|
|
7
9
|
import { MIGRATIONS, MigrationRunner } from "./migrations.js";
|
|
10
|
+
/**
|
|
11
|
+
* Plain (non-vec0) tables holding derived data keyed by (org_id, qualified_name).
|
|
12
|
+
* Purged in deleteNode so a removed node leaves no orphan rows. The vec0 vector
|
|
13
|
+
* table is NOT here — it needs the sqlite-vec extension and is purged via the
|
|
14
|
+
* VectorStore. Snapshot tables are intentionally excluded (point-in-time
|
|
15
|
+
* history must survive a node's deletion).
|
|
16
|
+
*/
|
|
17
|
+
const DERIVED_QNAME_TABLES = [
|
|
18
|
+
"_sfgraph_snippets",
|
|
19
|
+
"_sfgraph_governor_risks",
|
|
20
|
+
"_sfgraph_findings",
|
|
21
|
+
"_sfgraph_dead_code_scores",
|
|
22
|
+
"_sfgraph_test_coverage",
|
|
23
|
+
];
|
|
24
|
+
// _sfgraph_node_fts is also derived per-qname, but its key columns are FTS5
|
|
25
|
+
// UNINDEXED (a WHERE on them scans the whole table), so it is purged by rowid
|
|
26
|
+
// through _sfgraph_node_fts_rowid instead — see deleteNodeFts.
|
|
27
|
+
/** How long a write waits for another connection's lock (rel-05). */
|
|
28
|
+
export const BUSY_TIMEOUT_MS = 5000;
|
|
29
|
+
/**
|
|
30
|
+
* Page cache per connection, in KiB (`cache_size = -N`). Was 200 MB: whole-graph
|
|
31
|
+
* phases (resolvers, audit, deadcode) fill it on a large org DB (1.4 GB), and
|
|
32
|
+
* the pages stay resident. 64 MB kept a large Vlocity repo rebuild's wall time
|
|
33
|
+
* unchanged (3 interleaved runs each: 55.0 s at 200 MB vs 53.1 s).
|
|
34
|
+
*/
|
|
35
|
+
export const SQLITE_CACHE_SIZE_KIB = 64_000;
|
|
36
|
+
/** Memory-mapped I/O window per connection, in bytes (was 256 MiB; same measurement). */
|
|
37
|
+
export const SQLITE_MMAP_SIZE_BYTES = 64 * 1024 * 1024;
|
|
38
|
+
/** Members per bulk transaction (beginBulk/endBulk). */
|
|
39
|
+
export const BULK_TXN_MEMBERS = 2000;
|
|
40
|
+
/**
|
|
41
|
+
* Live-graph write batches (beginBatch): members per transaction, and the
|
|
42
|
+
* longest one transaction holds the write lock. A non-rebuild ingest writes
|
|
43
|
+
* into a graph that MCP servers may be reading (and, rarely, writing): readers
|
|
44
|
+
* are never blocked in WAL mode, and another writer waits at most about this
|
|
45
|
+
* long, far inside BUSY_TIMEOUT_MS.
|
|
46
|
+
*/
|
|
47
|
+
export const BATCH_TXN_MEMBERS = 250;
|
|
48
|
+
export const BATCH_TXN_MAX_MS = 200;
|
|
49
|
+
/** Edge tables per UNION ALL statement (SQLite's default compound-SELECT cap is 500). */
|
|
50
|
+
const MAX_UNION_ARMS = 200;
|
|
51
|
+
/** Unit separator; cannot occur in a qname or org id. */
|
|
52
|
+
const KSEP = String.fromCharCode(31);
|
|
53
|
+
/** Canonical secondary-index DDL of a node / edge table. The single source for
|
|
54
|
+
* ensureNodeTable / ensureEdgeTable and the post-crash heal in init(), so a
|
|
55
|
+
* healed index has byte-identical sqlite_master SQL (SQLite stores it without
|
|
56
|
+
* the IF NOT EXISTS and the trailing semicolon). */
|
|
57
|
+
function nodeIndexDdl(tbl) {
|
|
58
|
+
return `CREATE INDEX IF NOT EXISTS ${tbl}_org ON ${tbl}(org_id);`;
|
|
59
|
+
}
|
|
60
|
+
function edgeIndexDdl(tbl) {
|
|
61
|
+
return `CREATE INDEX IF NOT EXISTS ${tbl}_rev ON ${tbl}(org_id, dst_qname);`;
|
|
62
|
+
}
|
|
63
|
+
/** Durable marker of an open bulk window (one row). Written before any DROP
|
|
64
|
+
* INDEX and cleared only once endBulk has rebuilt the indexes, so a process
|
|
65
|
+
* that dies inside the window is detected and healed by the next init(). */
|
|
66
|
+
const BULK_MARKER_DDL = `CREATE TABLE IF NOT EXISTS _sfgraph_bulk_window (
|
|
67
|
+
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
68
|
+
pid INTEGER NOT NULL,
|
|
69
|
+
started_at INTEGER NOT NULL,
|
|
70
|
+
host TEXT
|
|
71
|
+
)`;
|
|
72
|
+
function chunks(xs, n) {
|
|
73
|
+
const out = [];
|
|
74
|
+
for (let i = 0; i < xs.length; i += n)
|
|
75
|
+
out.push(xs.slice(i, i + n));
|
|
76
|
+
return out;
|
|
77
|
+
}
|
|
78
|
+
/** ASCII-only lowercase — exactly what SQLite's NOCASE collation folds. */
|
|
79
|
+
function nocaseKey(s) {
|
|
80
|
+
// toLowerCase is exactly the ASCII fold on an ASCII-only string (the common
|
|
81
|
+
// case, and ~10x cheaper than a replace callback); non-ASCII letters must
|
|
82
|
+
// stay as they are, so those strings fold A-Z only.
|
|
83
|
+
for (let i = 0; i < s.length; i++) {
|
|
84
|
+
if (s.charCodeAt(i) > 0x7f)
|
|
85
|
+
return s.replace(/[A-Z]/g, (c) => c.toLowerCase());
|
|
86
|
+
}
|
|
87
|
+
return s.toLowerCase();
|
|
88
|
+
}
|
|
8
89
|
function rowToNode(row, label) {
|
|
9
90
|
return {
|
|
10
91
|
orgId: asOrgId(row.org_id),
|
|
@@ -28,18 +109,115 @@ function rowToEdge(row, relType) {
|
|
|
28
109
|
lastSeenAt: row.last_seen_at,
|
|
29
110
|
};
|
|
30
111
|
}
|
|
112
|
+
/**
|
|
113
|
+
* rel-02: `ownerIsStale` may spawn `ps` to rule out pid reuse; readers call
|
|
114
|
+
* getSyncStatus on every MCP tool call, so verdicts are memoised briefly per
|
|
115
|
+
* recorded owner. A dead owner never comes back, so a stale verdict is final.
|
|
116
|
+
*/
|
|
117
|
+
const SYNC_OWNER_TTL_MS = 30_000;
|
|
118
|
+
const syncOwnerVerdicts = new Map();
|
|
119
|
+
function syncOwnerIsStale(pid, host, startMs) {
|
|
120
|
+
if (pid === process.pid && isThisHost(host))
|
|
121
|
+
return false;
|
|
122
|
+
const key = `${host ?? ""}|${pid}|${startMs ?? ""}`;
|
|
123
|
+
const now = Date.now();
|
|
124
|
+
const hit = syncOwnerVerdicts.get(key);
|
|
125
|
+
if (hit && (hit.stale || now - hit.at < SYNC_OWNER_TTL_MS))
|
|
126
|
+
return hit.stale;
|
|
127
|
+
const stale = ownerIsStale({ pid, host, procStartMs: startMs });
|
|
128
|
+
if (syncOwnerVerdicts.size > 256)
|
|
129
|
+
syncOwnerVerdicts.clear();
|
|
130
|
+
syncOwnerVerdicts.set(key, { stale, at: now });
|
|
131
|
+
return stale;
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* review R1-1: whether the graph file at `dbPath` is a finished build that a
|
|
135
|
+
* reader may switch to. `ingest --rebuild` creates the new file at dbPath and
|
|
136
|
+
* fills it for minutes (indexes dropped until endBulk), so a reader that sees
|
|
137
|
+
* a new inode must keep its old connection until the new file:
|
|
138
|
+
* - has no bulk-load window whose owner is alive,
|
|
139
|
+
* - has an `_sfgraph_orgs` row (for `orgId`, or every row when omitted), and
|
|
140
|
+
* each such row has a completed sync (`last_synced_at`) and is not in
|
|
141
|
+
* progress with a live owner (same rule as getSyncStatus, rel-02).
|
|
142
|
+
* Opens the file read-only and never writes.
|
|
143
|
+
*/
|
|
144
|
+
export function graphFileReadiness(dbPath, orgId) {
|
|
145
|
+
let db;
|
|
146
|
+
try {
|
|
147
|
+
db = new Database(dbPath, { readonly: true, fileMustExist: true });
|
|
148
|
+
}
|
|
149
|
+
catch {
|
|
150
|
+
return { ready: false, reason: "the file cannot be opened" };
|
|
151
|
+
}
|
|
152
|
+
try {
|
|
153
|
+
const has = (t) => db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(t) !==
|
|
154
|
+
undefined;
|
|
155
|
+
if (has("_sfgraph_bulk_window")) {
|
|
156
|
+
const m = db.prepare("SELECT * FROM _sfgraph_bulk_window WHERE id = 1").get();
|
|
157
|
+
if (m) {
|
|
158
|
+
const marker = { pid: m.pid, host: m.host ?? null, takenAtMs: m.started_at ?? null };
|
|
159
|
+
const ours = m.pid === process.pid && isThisHost(marker.host);
|
|
160
|
+
if (ours || !ownerIsStale(marker))
|
|
161
|
+
return { ready: false, reason: "a bulk load is running" };
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
if (!has("_sfgraph_orgs"))
|
|
165
|
+
return { ready: false, reason: "no _sfgraph_orgs table" };
|
|
166
|
+
const rows = (orgId === undefined
|
|
167
|
+
? db.prepare("SELECT * FROM _sfgraph_orgs").all()
|
|
168
|
+
: db.prepare("SELECT * FROM _sfgraph_orgs WHERE id = ?").all(orgId));
|
|
169
|
+
if (rows.length === 0)
|
|
170
|
+
return { ready: false, reason: "no org row yet" };
|
|
171
|
+
for (const r of rows) {
|
|
172
|
+
if (r.last_synced_at == null)
|
|
173
|
+
return { ready: false, reason: "no completed sync" };
|
|
174
|
+
if (r.sync_in_progress === 1 &&
|
|
175
|
+
(r.sync_owner_pid == null ||
|
|
176
|
+
!syncOwnerIsStale(r.sync_owner_pid, r.sync_owner_host ?? null, r.sync_owner_start_ms ?? null))) {
|
|
177
|
+
return { ready: false, reason: "a sync is in progress" };
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
return { ready: true };
|
|
181
|
+
}
|
|
182
|
+
catch {
|
|
183
|
+
return { ready: false, reason: "the file is not readable yet" };
|
|
184
|
+
}
|
|
185
|
+
finally {
|
|
186
|
+
try {
|
|
187
|
+
db.close();
|
|
188
|
+
}
|
|
189
|
+
catch {
|
|
190
|
+
/* ignore */
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
}
|
|
31
194
|
export class SqliteGraphStore {
|
|
32
|
-
|
|
195
|
+
conn;
|
|
196
|
+
/** Connections replaced after a file swap; closed with the store (review I2). */
|
|
197
|
+
retired = [];
|
|
198
|
+
/** `dev:ino` of dbPath for the open connection; null when not file-backed. */
|
|
199
|
+
fileId = null;
|
|
200
|
+
lastFileCheck = 0;
|
|
33
201
|
ownsDb;
|
|
34
202
|
opts;
|
|
35
203
|
nodeLabelCache = new Map();
|
|
36
204
|
edgeRelCache = new Map();
|
|
205
|
+
/** `PRAGMA data_version` when the caches were last loaded (rel-01). */
|
|
206
|
+
dataVersion = -1;
|
|
37
207
|
initialized = false;
|
|
38
208
|
closed = false;
|
|
209
|
+
bulk = null;
|
|
210
|
+
batch = null;
|
|
211
|
+
/** edgesAllTables' SQL per column, for the current edge tables. */
|
|
212
|
+
unionEdgeQueries = {};
|
|
213
|
+
/** priorOutgoingEdges' SQL for the current edge tables. */
|
|
214
|
+
priorEdgesQuery = null;
|
|
215
|
+
stmtCache = new Map();
|
|
216
|
+
ftsUpsertTx = null;
|
|
39
217
|
constructor(opts) {
|
|
40
218
|
this.opts = opts;
|
|
41
219
|
if (opts.db) {
|
|
42
|
-
this.
|
|
220
|
+
this.conn = opts.db;
|
|
43
221
|
this.ownsDb = false;
|
|
44
222
|
}
|
|
45
223
|
else {
|
|
@@ -49,7 +227,7 @@ export class SqliteGraphStore {
|
|
|
49
227
|
mkdirSync(dir, { recursive: true });
|
|
50
228
|
}
|
|
51
229
|
try {
|
|
52
|
-
this.
|
|
230
|
+
this.conn = new Database(opts.dbPath);
|
|
53
231
|
}
|
|
54
232
|
catch (e) {
|
|
55
233
|
const wrapped = wrapAbiError(e);
|
|
@@ -60,15 +238,29 @@ export class SqliteGraphStore {
|
|
|
60
238
|
this.ownsDb = true;
|
|
61
239
|
}
|
|
62
240
|
}
|
|
241
|
+
get db() {
|
|
242
|
+
return this.conn;
|
|
243
|
+
}
|
|
63
244
|
async init() {
|
|
64
245
|
if (this.initialized)
|
|
65
246
|
return;
|
|
247
|
+
this.fileId = this.readFileId();
|
|
248
|
+
this.setupConnection();
|
|
249
|
+
this.loadCaches();
|
|
250
|
+
this.healBulkWindow();
|
|
251
|
+
this.initialized = true;
|
|
252
|
+
}
|
|
253
|
+
/** Pragmas + pending migrations for a freshly opened connection. */
|
|
254
|
+
setupConnection() {
|
|
255
|
+
// rel-05: explicit, not better-sqlite3's implicit default. A write that
|
|
256
|
+
// still finds the lock held after this throws SQLITE_BUSY.
|
|
257
|
+
this.db.pragma(`busy_timeout = ${BUSY_TIMEOUT_MS}`);
|
|
66
258
|
this.db.pragma("journal_mode = WAL");
|
|
67
259
|
this.db.pragma("synchronous = NORMAL");
|
|
68
260
|
this.db.pragma("foreign_keys = ON");
|
|
69
261
|
this.db.pragma("temp_store = MEMORY");
|
|
70
|
-
this.db.pragma(
|
|
71
|
-
this.db.pragma(
|
|
262
|
+
this.db.pragma(`mmap_size = ${SQLITE_MMAP_SIZE_BYTES}`);
|
|
263
|
+
this.db.pragma(`cache_size = -${SQLITE_CACHE_SIZE_KIB}`);
|
|
72
264
|
// Smaller WAL auto-checkpoint threshold (default 1000 pages = ~4MB).
|
|
73
265
|
// With large ingests producing 10-20K inserts during the object phase,
|
|
74
266
|
// the WAL hits the default threshold right around the 30-second mark
|
|
@@ -88,8 +280,74 @@ export class SqliteGraphStore {
|
|
|
88
280
|
retainBackups: this.opts.retainBackups ?? 5,
|
|
89
281
|
});
|
|
90
282
|
runner.applyPending();
|
|
91
|
-
|
|
92
|
-
|
|
283
|
+
}
|
|
284
|
+
readFileId() {
|
|
285
|
+
if (!this.ownsDb || this.opts.dbPath === ":memory:")
|
|
286
|
+
return null;
|
|
287
|
+
try {
|
|
288
|
+
const st = statSync(this.opts.dbPath);
|
|
289
|
+
return `${st.dev}:${st.ino}`;
|
|
290
|
+
}
|
|
291
|
+
catch {
|
|
292
|
+
return null;
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
/**
|
|
296
|
+
* review I2: `ingest --rebuild` renames dbPath away and builds a new file.
|
|
297
|
+
* This connection still holds the old inode, whose data_version never
|
|
298
|
+
* changes again, so compare dbPath's dev/ino and reopen when it differs.
|
|
299
|
+
* The old connection stays open until close(): other holders (shared
|
|
300
|
+
* snapshot/vector stores) may still have statements on it.
|
|
301
|
+
*/
|
|
302
|
+
checkFileSwap() {
|
|
303
|
+
if (this.fileId === null)
|
|
304
|
+
return;
|
|
305
|
+
const every = this.opts.fileCheckIntervalMs ?? 1000;
|
|
306
|
+
if (every < 0)
|
|
307
|
+
return;
|
|
308
|
+
const now = Date.now();
|
|
309
|
+
if (every > 0 && now - this.lastFileCheck < every)
|
|
310
|
+
return;
|
|
311
|
+
this.lastFileCheck = now;
|
|
312
|
+
if (this.bulk || this.batch || this.conn.inTransaction)
|
|
313
|
+
return;
|
|
314
|
+
const id = this.readFileId();
|
|
315
|
+
if (id === null || id === this.fileId)
|
|
316
|
+
return;
|
|
317
|
+
// review R1-1: keep serving the old (complete) graph until the new file's
|
|
318
|
+
// build has finished; the old inode stays readable after rename/unlink.
|
|
319
|
+
if (!graphFileReadiness(this.opts.dbPath).ready)
|
|
320
|
+
return;
|
|
321
|
+
let next;
|
|
322
|
+
try {
|
|
323
|
+
next = new Database(this.opts.dbPath);
|
|
324
|
+
}
|
|
325
|
+
catch {
|
|
326
|
+
return; // mid-swap; retry on a later read
|
|
327
|
+
}
|
|
328
|
+
const prev = this.conn;
|
|
329
|
+
this.conn = next;
|
|
330
|
+
this.stmtCache.clear();
|
|
331
|
+
this.ftsUpsertTx = null;
|
|
332
|
+
try {
|
|
333
|
+
this.setupConnection();
|
|
334
|
+
this.loadCaches();
|
|
335
|
+
}
|
|
336
|
+
catch {
|
|
337
|
+
// e.g. the rebuild still holds the write lock: keep the old graph for now.
|
|
338
|
+
this.conn = prev;
|
|
339
|
+
this.stmtCache.clear();
|
|
340
|
+
try {
|
|
341
|
+
next.close();
|
|
342
|
+
}
|
|
343
|
+
catch {
|
|
344
|
+
/* ignore */
|
|
345
|
+
}
|
|
346
|
+
this.loadCaches();
|
|
347
|
+
return;
|
|
348
|
+
}
|
|
349
|
+
this.retired.push(prev);
|
|
350
|
+
this.fileId = id;
|
|
93
351
|
}
|
|
94
352
|
/**
|
|
95
353
|
* Flush WAL to the main DB file with a PASSIVE checkpoint. Cheap (no
|
|
@@ -114,6 +372,22 @@ export class SqliteGraphStore {
|
|
|
114
372
|
async close() {
|
|
115
373
|
if (this.closed)
|
|
116
374
|
return;
|
|
375
|
+
if (this.bulk) {
|
|
376
|
+
try {
|
|
377
|
+
this.endBulk();
|
|
378
|
+
}
|
|
379
|
+
catch {
|
|
380
|
+
/* best-effort; the next init() heals any missing indexes */
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
if (this.batch) {
|
|
384
|
+
try {
|
|
385
|
+
this.endBatch();
|
|
386
|
+
}
|
|
387
|
+
catch {
|
|
388
|
+
/* best-effort */
|
|
389
|
+
}
|
|
390
|
+
}
|
|
117
391
|
this.closed = true;
|
|
118
392
|
if (this.ownsDb) {
|
|
119
393
|
try {
|
|
@@ -131,29 +405,75 @@ export class SqliteGraphStore {
|
|
|
131
405
|
catch {
|
|
132
406
|
// already closed by another path — idempotent
|
|
133
407
|
}
|
|
408
|
+
for (const old of this.retired.splice(0)) {
|
|
409
|
+
try {
|
|
410
|
+
old.close();
|
|
411
|
+
}
|
|
412
|
+
catch {
|
|
413
|
+
/* ignore */
|
|
414
|
+
}
|
|
415
|
+
}
|
|
134
416
|
}
|
|
135
417
|
}
|
|
136
418
|
loadCaches() {
|
|
137
|
-
const labels = this.
|
|
138
|
-
|
|
139
|
-
.all();
|
|
419
|
+
const labels = this.stmt("SELECT label, table_name FROM _sfgraph_node_labels").all();
|
|
420
|
+
const nodeLabels = new Map();
|
|
140
421
|
for (const row of labels)
|
|
141
|
-
|
|
142
|
-
const edges = this.
|
|
143
|
-
|
|
144
|
-
.all();
|
|
422
|
+
nodeLabels.set(row.label, row.table_name);
|
|
423
|
+
const edges = this.stmt("SELECT rel_type, table_name FROM _sfgraph_edge_types").all();
|
|
424
|
+
const edgeRels = new Map();
|
|
145
425
|
for (const row of edges)
|
|
146
|
-
|
|
426
|
+
edgeRels.set(row.rel_type, row.table_name);
|
|
427
|
+
this.nodeLabelCache = nodeLabels;
|
|
428
|
+
this.edgeRelCache = edgeRels;
|
|
429
|
+
this.dataVersion = this.readDataVersion();
|
|
430
|
+
}
|
|
431
|
+
readDataVersion() {
|
|
432
|
+
return this.stmt("PRAGMA data_version").get().data_version;
|
|
433
|
+
}
|
|
434
|
+
/**
|
|
435
|
+
* rel-01: the label / rel-type registries can grow under a long-lived
|
|
436
|
+
* reader (an ingest in another process creates a new node or edge table).
|
|
437
|
+
* `PRAGMA data_version` changes only when ANOTHER connection commits, so
|
|
438
|
+
* this is one cheap read per call and a reload only after a foreign write.
|
|
439
|
+
* Writes through this connection keep the caches current themselves.
|
|
440
|
+
*/
|
|
441
|
+
syncCaches() {
|
|
442
|
+
if (this.closed)
|
|
443
|
+
return;
|
|
444
|
+
this.checkFileSwap();
|
|
445
|
+
if (this.readDataVersion() !== this.dataVersion)
|
|
446
|
+
this.loadCaches();
|
|
447
|
+
}
|
|
448
|
+
/** Cached prepared statement (re-preparing per call cost ~14 s per full ingest). */
|
|
449
|
+
stmt(sql) {
|
|
450
|
+
let st = this.stmtCache.get(sql);
|
|
451
|
+
if (!st) {
|
|
452
|
+
st = this.db.prepare(sql);
|
|
453
|
+
this.stmtCache.set(sql, st);
|
|
454
|
+
}
|
|
455
|
+
return st;
|
|
456
|
+
}
|
|
457
|
+
/** Run a table's secondary-index DDL now, or defer it to endBulk. */
|
|
458
|
+
createIndex(name, table, sql) {
|
|
459
|
+
if (this.bulk) {
|
|
460
|
+
this.bulk.deferredIndexes.push({ name, table, sql });
|
|
461
|
+
return;
|
|
462
|
+
}
|
|
463
|
+
this.db.exec(sql);
|
|
147
464
|
}
|
|
148
465
|
ensureNodeTable(label) {
|
|
149
466
|
const cached = this.nodeLabelCache.get(label);
|
|
150
467
|
if (cached)
|
|
151
468
|
return cached;
|
|
152
469
|
const tbl = nodeTableName(label);
|
|
470
|
+
// qualified_name is COLLATE NOCASE: Salesforce API names are case-insensitive
|
|
471
|
+
// (Account == account), so the PK dedups and all `= ?` lookups fold case while
|
|
472
|
+
// the original display string is preserved. See migration 9.
|
|
153
473
|
this.db.exec(`
|
|
154
474
|
CREATE TABLE IF NOT EXISTS ${tbl} (
|
|
155
475
|
org_id TEXT NOT NULL,
|
|
156
|
-
qualified_name TEXT NOT NULL,
|
|
476
|
+
qualified_name TEXT NOT NULL COLLATE NOCASE,
|
|
157
477
|
attributes TEXT NOT NULL,
|
|
158
478
|
source_hash TEXT NOT NULL,
|
|
159
479
|
first_seen_at INTEGER NOT NULL,
|
|
@@ -161,8 +481,8 @@ export class SqliteGraphStore {
|
|
|
161
481
|
last_modified_at INTEGER NOT NULL,
|
|
162
482
|
PRIMARY KEY(org_id, qualified_name)
|
|
163
483
|
);
|
|
164
|
-
CREATE INDEX IF NOT EXISTS ${tbl}_org ON ${tbl}(org_id);
|
|
165
484
|
`);
|
|
485
|
+
this.createIndex(`${tbl}_org`, tbl, nodeIndexDdl(tbl));
|
|
166
486
|
this.db
|
|
167
487
|
.prepare("INSERT OR IGNORE INTO _sfgraph_node_labels(label, table_name, created_at) VALUES (?, ?, ?)")
|
|
168
488
|
.run(label, tbl, Date.now());
|
|
@@ -174,18 +494,21 @@ export class SqliteGraphStore {
|
|
|
174
494
|
if (cached)
|
|
175
495
|
return cached;
|
|
176
496
|
const tbl = edgeTableName(relType);
|
|
497
|
+
// src/dst qnames COLLATE NOCASE so a reference's case (e.g. SOQL `from
|
|
498
|
+
// account`) matches the definition node's qname and edges dedup/join
|
|
499
|
+
// case-insensitively. See migration 9.
|
|
177
500
|
this.db.exec(`
|
|
178
501
|
CREATE TABLE IF NOT EXISTS ${tbl} (
|
|
179
502
|
org_id TEXT NOT NULL,
|
|
180
|
-
src_qname TEXT NOT NULL,
|
|
181
|
-
dst_qname TEXT NOT NULL,
|
|
503
|
+
src_qname TEXT NOT NULL COLLATE NOCASE,
|
|
504
|
+
dst_qname TEXT NOT NULL COLLATE NOCASE,
|
|
182
505
|
attributes TEXT NOT NULL,
|
|
183
506
|
first_seen_at INTEGER NOT NULL,
|
|
184
507
|
last_seen_at INTEGER NOT NULL,
|
|
185
508
|
PRIMARY KEY(org_id, src_qname, dst_qname)
|
|
186
509
|
);
|
|
187
|
-
CREATE INDEX IF NOT EXISTS ${tbl}_rev ON ${tbl}(org_id, dst_qname);
|
|
188
510
|
`);
|
|
511
|
+
this.createIndex(`${tbl}_rev`, tbl, edgeIndexDdl(tbl));
|
|
189
512
|
this.db
|
|
190
513
|
.prepare("INSERT OR IGNORE INTO _sfgraph_edge_types(rel_type, table_name, created_at) VALUES (?, ?, ?)")
|
|
191
514
|
.run(relType, tbl, Date.now());
|
|
@@ -224,7 +547,88 @@ export class SqliteGraphStore {
|
|
|
224
547
|
}
|
|
225
548
|
this.db.prepare("UPDATE _sfgraph_orgs SET last_synced_at = ? WHERE id = ?").run(ts, orgId);
|
|
226
549
|
}
|
|
550
|
+
// ─── W1.5-08: sync-generation lifecycle ────────────────────────────────────
|
|
551
|
+
// Three helpers (started / complete / failed) plus a getter. The shape is
|
|
552
|
+
// designed so concurrent MCP readers can answer "is the graph being
|
|
553
|
+
// rewritten right now?" cheaply (one SELECT) without taking a global lock.
|
|
554
|
+
// Generation only advances on success — failed ingests are invisible to
|
|
555
|
+
// the counter, matching how readers would expect "the graph has been
|
|
556
|
+
// updated" to be a synonym of "an ingest completed cleanly".
|
|
557
|
+
markSyncStarted(orgId, startedAtIso) {
|
|
558
|
+
// Validate the ISO early — better to fail loud at start than to write a
|
|
559
|
+
// garbage timestamp and confuse the staleness block readers see.
|
|
560
|
+
if (!Number.isFinite(Date.parse(startedAtIso))) {
|
|
561
|
+
throw new StorageError(`markSyncStarted: invalid ISO timestamp ${JSON.stringify(startedAtIso)}`);
|
|
562
|
+
}
|
|
563
|
+
// rel-02: record the owner so a reader can tell this run from one that
|
|
564
|
+
// was killed without clearing the flag.
|
|
565
|
+
this.db
|
|
566
|
+
.prepare(`UPDATE _sfgraph_orgs
|
|
567
|
+
SET sync_in_progress = 1, sync_started_at = ?,
|
|
568
|
+
sync_owner_pid = ?, sync_owner_host = ?, sync_owner_start_ms = ?
|
|
569
|
+
WHERE id = ?`)
|
|
570
|
+
.run(startedAtIso, process.pid, thisHost(), ownStartMs(), orgId);
|
|
571
|
+
}
|
|
572
|
+
markSyncComplete(orgId, completedAtIso) {
|
|
573
|
+
const ts = Date.parse(completedAtIso);
|
|
574
|
+
if (!Number.isFinite(ts)) {
|
|
575
|
+
throw new StorageError(`markSyncComplete: invalid ISO timestamp ${JSON.stringify(completedAtIso)}`);
|
|
576
|
+
}
|
|
577
|
+
// Single transaction so concurrent readers never observe a half-applied
|
|
578
|
+
// state (e.g. generation incremented but in_progress still 1).
|
|
579
|
+
const apply = this.db.transaction(() => {
|
|
580
|
+
this.db
|
|
581
|
+
.prepare(`UPDATE _sfgraph_orgs
|
|
582
|
+
SET sync_generation = sync_generation + 1,
|
|
583
|
+
sync_in_progress = 0,
|
|
584
|
+
sync_started_at = NULL,
|
|
585
|
+
sync_owner_pid = NULL, sync_owner_host = NULL, sync_owner_start_ms = NULL,
|
|
586
|
+
last_synced_at = ?
|
|
587
|
+
WHERE id = ?`)
|
|
588
|
+
.run(ts, orgId);
|
|
589
|
+
});
|
|
590
|
+
apply();
|
|
591
|
+
}
|
|
592
|
+
markSyncFailed(orgId) {
|
|
593
|
+
// Generation deliberately untouched — a failed ingest is not a new
|
|
594
|
+
// version of the graph, so readers polling the counter should not be
|
|
595
|
+
// tricked into thinking data refreshed.
|
|
596
|
+
this.db
|
|
597
|
+
.prepare(`UPDATE _sfgraph_orgs
|
|
598
|
+
SET sync_in_progress = 0, sync_started_at = NULL,
|
|
599
|
+
sync_owner_pid = NULL, sync_owner_host = NULL, sync_owner_start_ms = NULL
|
|
600
|
+
WHERE id = ?`)
|
|
601
|
+
.run(orgId);
|
|
602
|
+
}
|
|
603
|
+
getSyncStatus(orgId) {
|
|
604
|
+
const row = this.db
|
|
605
|
+
.prepare(`SELECT sync_generation, sync_in_progress, sync_started_at, last_synced_at,
|
|
606
|
+
sync_owner_pid, sync_owner_host, sync_owner_start_ms
|
|
607
|
+
FROM _sfgraph_orgs WHERE id = ?`)
|
|
608
|
+
.get(orgId);
|
|
609
|
+
if (!row) {
|
|
610
|
+
// Org row not present yet (e.g. tool fired against an MCP context
|
|
611
|
+
// whose DB was just provisioned). Return sane defaults so callers
|
|
612
|
+
// don't have to null-check every field.
|
|
613
|
+
return { generation: 0, in_progress: false, started_at: null, last_sync_at: null };
|
|
614
|
+
}
|
|
615
|
+
// rel-02: a flag whose recorded owner is gone (killed before it could
|
|
616
|
+
// clear it) is reported as not in progress. Report-only: the row is left
|
|
617
|
+
// for the next ingest to overwrite. Legacy rows with no owner are trusted.
|
|
618
|
+
const flagged = (row.sync_in_progress ?? 0) === 1;
|
|
619
|
+
const dead = flagged &&
|
|
620
|
+
row.sync_owner_pid != null &&
|
|
621
|
+
syncOwnerIsStale(row.sync_owner_pid, row.sync_owner_host, row.sync_owner_start_ms);
|
|
622
|
+
const inProgress = flagged && !dead;
|
|
623
|
+
return {
|
|
624
|
+
generation: row.sync_generation ?? 0,
|
|
625
|
+
in_progress: inProgress,
|
|
626
|
+
started_at: inProgress ? (row.sync_started_at ?? null) : null,
|
|
627
|
+
last_sync_at: row.last_synced_at ?? null,
|
|
628
|
+
};
|
|
629
|
+
}
|
|
227
630
|
deleteNode(orgId, qname) {
|
|
631
|
+
this.syncCaches();
|
|
228
632
|
const apply = this.db.transaction(() => {
|
|
229
633
|
const idx = this.db
|
|
230
634
|
.prepare("SELECT label FROM _sfgraph_node_index WHERE org_id = ? AND qualified_name = ?")
|
|
@@ -240,15 +644,29 @@ export class SqliteGraphStore {
|
|
|
240
644
|
.prepare("DELETE FROM _sfgraph_node_index WHERE org_id = ? AND qualified_name = ?")
|
|
241
645
|
.run(orgId, qname);
|
|
242
646
|
}
|
|
647
|
+
// Purge derived per-qname rows so a deleted node leaves no orphans that
|
|
648
|
+
// would surface in explain_code (snippet) or the audits (findings /
|
|
649
|
+
// governor risks / dead-code / coverage). Run unconditionally — derived
|
|
650
|
+
// rows can outlive a missing index entry. The vec0 vector lives in a
|
|
651
|
+
// separate extension-loaded connection and is purged by the caller via
|
|
652
|
+
// VectorStore.deleteNodeVector. Each table is keyed (org_id, qualified_name, …).
|
|
653
|
+
for (const t of DERIVED_QNAME_TABLES) {
|
|
654
|
+
this.stmt(`DELETE FROM ${t} WHERE org_id = ? AND qualified_name = ?`).run(orgId, qname);
|
|
655
|
+
}
|
|
656
|
+
this.deleteNodeFts(String(orgId), String(qname));
|
|
243
657
|
});
|
|
244
658
|
apply();
|
|
245
659
|
}
|
|
246
660
|
deleteEdgesFor(orgId, qname) {
|
|
661
|
+
this.syncCaches();
|
|
247
662
|
const apply = this.db.transaction(() => {
|
|
663
|
+
// Two statements, not `src = ? OR dst = ?`: SQLite plans the OR as a scan of the org's
|
|
664
|
+
// whole slice of the table (one `_rev (org_id=?)` search). Split, each side uses an
|
|
665
|
+
// index: the PK `(org_id, src_qname, dst_qname)` and `_rev (org_id, dst_qname)`.
|
|
666
|
+
// About 17x faster on a 300k-edge org; the rows removed are the same.
|
|
248
667
|
for (const tbl of this.edgeRelCache.values()) {
|
|
249
|
-
this.
|
|
250
|
-
|
|
251
|
-
.run(orgId, qname, qname);
|
|
668
|
+
this.stmt(`DELETE FROM ${tbl} WHERE org_id = ? AND src_qname = ?`).run(orgId, qname);
|
|
669
|
+
this.stmt(`DELETE FROM ${tbl} WHERE org_id = ? AND dst_qname = ?`).run(orgId, qname);
|
|
252
670
|
}
|
|
253
671
|
});
|
|
254
672
|
apply();
|
|
@@ -256,6 +674,7 @@ export class SqliteGraphStore {
|
|
|
256
674
|
mergeNodes(facts) {
|
|
257
675
|
if (facts.length === 0)
|
|
258
676
|
return { inserted: 0, updated: 0, unchanged: 0 };
|
|
677
|
+
this.syncCaches();
|
|
259
678
|
const buckets = new Map();
|
|
260
679
|
for (const f of facts) {
|
|
261
680
|
validateLabel(f.label);
|
|
@@ -267,19 +686,38 @@ export class SqliteGraphStore {
|
|
|
267
686
|
let updated = 0;
|
|
268
687
|
let unchanged = 0;
|
|
269
688
|
const apply = this.db.transaction(() => {
|
|
689
|
+
// Look up a node's currently-recorded label (across all per-label tables)
|
|
690
|
+
// so we can detect a label TRANSITION (e.g. ApexMethod → TestMethod when a
|
|
691
|
+
// method gains @isTest, class ↔ interface). On transition the node moves
|
|
692
|
+
// to a new per-label table; the stale old-label row must be deleted, or
|
|
693
|
+
// the detect-deletions sweep sees it as unreferenced and deletes the LIVE
|
|
694
|
+
// node + its edges (deleteNode resolves the table via the index, which now
|
|
695
|
+
// points at the new label). Edges are NOT touched — the qname is unchanged.
|
|
696
|
+
const indexLabelStmt = this.stmt("SELECT label FROM _sfgraph_node_index WHERE org_id = ? AND qualified_name = ?");
|
|
697
|
+
const deleteOldRow = (oldLabel, orgId, qname) => {
|
|
698
|
+
const oldTbl = this.ensureNodeTable(oldLabel);
|
|
699
|
+
this.stmt(`DELETE FROM ${oldTbl} WHERE org_id = ? AND qualified_name = ?`).run(orgId, qname);
|
|
700
|
+
};
|
|
270
701
|
for (const [label, bucket] of buckets) {
|
|
271
702
|
const tbl = this.ensureNodeTable(label);
|
|
272
|
-
const selectStmt = this.
|
|
273
|
-
const insertStmt = this.
|
|
703
|
+
const selectStmt = this.stmt(`SELECT source_hash, first_seen_at FROM ${tbl} WHERE org_id = ? AND qualified_name = ?`);
|
|
704
|
+
const insertStmt = this.stmt(`INSERT INTO ${tbl}(org_id, qualified_name, attributes, source_hash, first_seen_at, last_seen_at, last_modified_at)
|
|
274
705
|
VALUES (?, ?, ?, ?, ?, ?, ?)`);
|
|
275
|
-
const updateChangedStmt = this.
|
|
276
|
-
const updateUnchangedStmt = this.
|
|
277
|
-
const indexStmt = this.
|
|
706
|
+
const updateChangedStmt = this.stmt(`UPDATE ${tbl} SET attributes = ?, source_hash = ?, last_seen_at = ?, last_modified_at = ? WHERE org_id = ? AND qualified_name = ?`);
|
|
707
|
+
const updateUnchangedStmt = this.stmt(`UPDATE ${tbl} SET last_seen_at = ? WHERE org_id = ? AND qualified_name = ?`);
|
|
708
|
+
const indexStmt = this.stmt(`INSERT INTO _sfgraph_node_index(org_id, qualified_name, label) VALUES (?, ?, ?)
|
|
278
709
|
ON CONFLICT(org_id, qualified_name) DO UPDATE SET label=excluded.label`);
|
|
279
710
|
for (const fact of bucket) {
|
|
280
711
|
const existing = selectStmt.get(fact.orgId, fact.qualifiedName);
|
|
281
712
|
const attrsJson = JSON.stringify(fact.attributes);
|
|
282
713
|
if (!existing) {
|
|
714
|
+
// Not present under THIS label. If the index has it under a
|
|
715
|
+
// DIFFERENT label, it's a label transition — purge the stale old row
|
|
716
|
+
// before inserting the new one (see the transaction-level comment).
|
|
717
|
+
const prior = indexLabelStmt.get(fact.orgId, fact.qualifiedName);
|
|
718
|
+
if (prior && prior.label !== label) {
|
|
719
|
+
deleteOldRow(prior.label, String(fact.orgId), String(fact.qualifiedName));
|
|
720
|
+
}
|
|
283
721
|
insertStmt.run(fact.orgId, fact.qualifiedName, attrsJson, fact.sourceHash, fact.firstSeenAt, fact.lastSeenAt, fact.lastModifiedAt);
|
|
284
722
|
indexStmt.run(fact.orgId, fact.qualifiedName, label);
|
|
285
723
|
inserted += 1;
|
|
@@ -298,8 +736,12 @@ export class SqliteGraphStore {
|
|
|
298
736
|
apply();
|
|
299
737
|
return { inserted, updated, unchanged };
|
|
300
738
|
}
|
|
301
|
-
mergeEdges(facts) {
|
|
302
|
-
|
|
739
|
+
mergeEdges(facts, opts) {
|
|
740
|
+
this.syncCaches();
|
|
741
|
+
const extras = opts?.reconcileSources === true && opts.reconcileExtraSrcs?.qnames.length
|
|
742
|
+
? opts.reconcileExtraSrcs
|
|
743
|
+
: undefined;
|
|
744
|
+
if (facts.length === 0 && !extras)
|
|
303
745
|
return { inserted: 0, updated: 0, unchanged: 0 };
|
|
304
746
|
const buckets = new Map();
|
|
305
747
|
for (const f of facts) {
|
|
@@ -308,55 +750,276 @@ export class SqliteGraphStore {
|
|
|
308
750
|
arr.push(f);
|
|
309
751
|
buckets.set(f.relType, arr);
|
|
310
752
|
}
|
|
753
|
+
// Reconciliation bookkeeping: the set of source qnames this batch is
|
|
754
|
+
// authoritative for, plus the exact (org|src|dst|rel) edges it carries, so
|
|
755
|
+
// we can prune any prior outgoing edge from those sources that is no longer
|
|
756
|
+
// emitted. The delimiter is a unit-separator that cannot occur in a qname.
|
|
757
|
+
const reconcile = opts?.reconcileSources === true;
|
|
758
|
+
const RSEP = String.fromCharCode(31);
|
|
759
|
+
// org -> set of source qnames this batch is authoritative for, plus the
|
|
760
|
+
// exact (org|src|dst|rel) edges it carries. Reconciliation prunes prior
|
|
761
|
+
// outgoing edges from those sources that the batch no longer emits.
|
|
762
|
+
const srcByOrg = new Map();
|
|
763
|
+
const batchEdgeKeys = new Set();
|
|
764
|
+
// Bulk fast path: in a fresh-org bulk window, a source this window has not
|
|
765
|
+
// written yet has no prior outgoing edges (the org was empty at beginBulk),
|
|
766
|
+
// so its reconcile probe — one SELECT per edge table per chunk — is skipped.
|
|
767
|
+
// Sources written earlier in the window are still probed (and pruned), so
|
|
768
|
+
// the result is identical to the slow path. Membership is decided BEFORE
|
|
769
|
+
// this batch's inserts are recorded.
|
|
770
|
+
const bulk = this.bulk;
|
|
771
|
+
const freshOrg = bulk?.freshOrg ?? null;
|
|
772
|
+
if (reconcile) {
|
|
773
|
+
for (const f of facts) {
|
|
774
|
+
const org = String(f.orgId);
|
|
775
|
+
const src = String(f.srcQualifiedName);
|
|
776
|
+
// NOCASE-folded, like the src/dst columns: a re-emit that differs only
|
|
777
|
+
// in case updates the stored row, so it must also keep it (corecorr-05).
|
|
778
|
+
batchEdgeKeys.add(`${org}${RSEP}${nocaseKey(src)}${RSEP}${nocaseKey(String(f.dstQualifiedName))}${RSEP}${f.relType}`);
|
|
779
|
+
let set = srcByOrg.get(org);
|
|
780
|
+
if (!set) {
|
|
781
|
+
set = new Set();
|
|
782
|
+
srcByOrg.set(org, set);
|
|
783
|
+
}
|
|
784
|
+
set.add(src);
|
|
785
|
+
}
|
|
786
|
+
if (extras) {
|
|
787
|
+
const org = String(extras.orgId);
|
|
788
|
+
let set = srcByOrg.get(org);
|
|
789
|
+
if (!set) {
|
|
790
|
+
set = new Set();
|
|
791
|
+
srcByOrg.set(org, set);
|
|
792
|
+
}
|
|
793
|
+
for (const q of extras.qnames)
|
|
794
|
+
set.add(q);
|
|
795
|
+
}
|
|
796
|
+
if (bulk && freshOrg !== null) {
|
|
797
|
+
const fresh = srcByOrg.get(freshOrg);
|
|
798
|
+
if (fresh) {
|
|
799
|
+
// Count per distinct NOCASE source, matching how SQLite would probe.
|
|
800
|
+
const seen = new Set();
|
|
801
|
+
for (const src of [...fresh]) {
|
|
802
|
+
const k = `${freshOrg}${KSEP}${nocaseKey(src)}`;
|
|
803
|
+
if (!bulk.edgeSrcs.has(k)) {
|
|
804
|
+
fresh.delete(src);
|
|
805
|
+
if (!seen.has(k))
|
|
806
|
+
bulk.stats.reconcileSkipped += 1;
|
|
807
|
+
}
|
|
808
|
+
else if (!seen.has(k)) {
|
|
809
|
+
bulk.stats.reconcileProbed += 1;
|
|
810
|
+
}
|
|
811
|
+
seen.add(k);
|
|
812
|
+
}
|
|
813
|
+
if (fresh.size === 0)
|
|
814
|
+
srcByOrg.delete(freshOrg);
|
|
815
|
+
}
|
|
816
|
+
}
|
|
817
|
+
}
|
|
818
|
+
if (bulk && freshOrg !== null) {
|
|
819
|
+
for (const f of facts) {
|
|
820
|
+
if (String(f.orgId) === freshOrg) {
|
|
821
|
+
bulk.edgeSrcs.add(`${freshOrg}${KSEP}${nocaseKey(String(f.srcQualifiedName))}`);
|
|
822
|
+
}
|
|
823
|
+
}
|
|
824
|
+
}
|
|
311
825
|
let inserted = 0;
|
|
312
826
|
let updated = 0;
|
|
313
827
|
let unchanged = 0;
|
|
828
|
+
let deleted = 0;
|
|
314
829
|
const apply = this.db.transaction(() => {
|
|
830
|
+
// Reconcile: read every prior outgoing edge of the owned sources once,
|
|
831
|
+
// across all edge tables (one UNION ALL statement per org), BEFORE this
|
|
832
|
+
// batch writes. It answers the per-fact existence probe for those
|
|
833
|
+
// sources (kept current as the batch inserts / updates, so a repeat
|
|
834
|
+
// within the batch sees the earlier write, as the probe would) and is
|
|
835
|
+
// the prune candidate set below. Rows this batch inserts are always in
|
|
836
|
+
// batchEdgeKeys, so pruning against the pre-write snapshot deletes
|
|
837
|
+
// exactly the rows the old post-write scan deleted.
|
|
838
|
+
const prior = new Map();
|
|
839
|
+
const priorRows = [];
|
|
840
|
+
const covered = new Set();
|
|
841
|
+
const deferredSeen = new Map();
|
|
842
|
+
if (reconcile && srcByOrg.size > 0) {
|
|
843
|
+
for (const [org, srcSet] of srcByOrg) {
|
|
844
|
+
for (const src of srcSet)
|
|
845
|
+
covered.add(`${org}${KSEP}${nocaseKey(src)}`);
|
|
846
|
+
for (const { relType, r } of this.priorOutgoingEdges(org, [...srcSet])) {
|
|
847
|
+
priorRows.push({ org, relType, r });
|
|
848
|
+
prior.set(`${org}${RSEP}${nocaseKey(r.s)}${RSEP}${nocaseKey(r.d)}${RSEP}${relType}`, {
|
|
849
|
+
attributes: r.a,
|
|
850
|
+
id: r.id,
|
|
851
|
+
});
|
|
852
|
+
}
|
|
853
|
+
}
|
|
854
|
+
}
|
|
315
855
|
for (const [relType, bucket] of buckets) {
|
|
316
856
|
const tbl = this.ensureEdgeTable(relType);
|
|
317
|
-
const selectStmt = this.
|
|
318
|
-
const insertStmt = this.
|
|
319
|
-
const updateChangedStmt = this.
|
|
320
|
-
const updateSeenStmt = this.
|
|
857
|
+
const selectStmt = this.stmt(`SELECT attributes, first_seen_at FROM ${tbl} WHERE org_id = ? AND src_qname = ? AND dst_qname = ?`);
|
|
858
|
+
const insertStmt = this.stmt(`INSERT INTO ${tbl}(org_id, src_qname, dst_qname, attributes, first_seen_at, last_seen_at) VALUES (?, ?, ?, ?, ?, ?)`);
|
|
859
|
+
const updateChangedStmt = this.stmt(`UPDATE ${tbl} SET attributes = ?, last_seen_at = ? WHERE org_id = ? AND src_qname = ? AND dst_qname = ?`);
|
|
860
|
+
const updateSeenStmt = this.stmt(`UPDATE ${tbl} SET last_seen_at = ? WHERE org_id = ? AND src_qname = ? AND dst_qname = ?`);
|
|
861
|
+
// M-3: a re-parsed source re-emits its guessed dst (`(?)` arity
|
|
862
|
+
// guess, `ApexClass:` interface name, formula hop guess); the edge a
|
|
863
|
+
// resolver rewrote it to is still present here (pruned below), so the
|
|
864
|
+
// fresh guess inherits that edge's first_seen_at. Probed only for
|
|
865
|
+
// sources this reconcile batch owns and that may have prior rows.
|
|
866
|
+
const rewrittenFromStmt = reconcile
|
|
867
|
+
? this.stmt(`SELECT MIN(first_seen_at) AS t FROM ${tbl} WHERE org_id = ? AND src_qname = ? AND (
|
|
868
|
+
json_extract(attributes, '$.arityResolvedFrom') = ? COLLATE NOCASE OR
|
|
869
|
+
json_extract(attributes, '$.retargetedFrom') = ? COLLATE NOCASE OR
|
|
870
|
+
json_extract(attributes, '$.hopResolvedFrom') = ? COLLATE NOCASE)`)
|
|
871
|
+
: null;
|
|
321
872
|
for (const fact of bucket) {
|
|
322
|
-
const
|
|
873
|
+
const org = String(fact.orgId);
|
|
874
|
+
const src = String(fact.srcQualifiedName);
|
|
875
|
+
const priorKey = covered.has(`${org}${KSEP}${nocaseKey(src)}`)
|
|
876
|
+
? `${org}${RSEP}${nocaseKey(src)}${RSEP}${nocaseKey(String(fact.dstQualifiedName))}${RSEP}${relType}`
|
|
877
|
+
: null;
|
|
878
|
+
const existing = (priorKey !== null
|
|
879
|
+
? prior.get(priorKey)
|
|
880
|
+
: selectStmt.get(fact.orgId, fact.srcQualifiedName, fact.dstQualifiedName));
|
|
323
881
|
const attrsJson = JSON.stringify(fact.attributes);
|
|
882
|
+
if (priorKey !== null && existing?.attributes !== attrsJson) {
|
|
883
|
+
prior.set(priorKey, existing?.id !== undefined
|
|
884
|
+
? { attributes: attrsJson, id: existing.id }
|
|
885
|
+
: { attributes: attrsJson });
|
|
886
|
+
}
|
|
324
887
|
if (!existing) {
|
|
325
|
-
|
|
888
|
+
let firstSeenAt = fact.firstSeenAt;
|
|
889
|
+
if (rewrittenFromStmt &&
|
|
890
|
+
srcByOrg.get(String(fact.orgId))?.has(String(fact.srcQualifiedName))) {
|
|
891
|
+
const dst = String(fact.dstQualifiedName);
|
|
892
|
+
if (dst.endsWith("(?)") ||
|
|
893
|
+
dst.startsWith("ApexClass:") ||
|
|
894
|
+
fact.attributes.hopRelationship != null) {
|
|
895
|
+
const row = rewrittenFromStmt.get(fact.orgId, fact.srcQualifiedName, dst, dst, dst);
|
|
896
|
+
if (row?.t != null && row.t < firstSeenAt)
|
|
897
|
+
firstSeenAt = row.t;
|
|
898
|
+
}
|
|
899
|
+
}
|
|
900
|
+
insertStmt.run(fact.orgId, fact.srcQualifiedName, fact.dstQualifiedName, attrsJson, firstSeenAt, fact.lastSeenAt);
|
|
326
901
|
inserted += 1;
|
|
327
902
|
}
|
|
328
903
|
else if (existing.attributes !== attrsJson) {
|
|
904
|
+
// Runs now, so an earlier deferred last_seen_at write is moot.
|
|
905
|
+
if (existing.id !== undefined)
|
|
906
|
+
deferredSeen.get(relType)?.delete(existing.id);
|
|
329
907
|
updateChangedStmt.run(attrsJson, fact.lastSeenAt, fact.orgId, fact.srcQualifiedName, fact.dstQualifiedName);
|
|
330
908
|
updated += 1;
|
|
331
909
|
}
|
|
910
|
+
else if (existing.id !== undefined) {
|
|
911
|
+
// A prior row re-emitted unchanged: its last_seen_at is written
|
|
912
|
+
// below, in one statement per table (Map order = last write wins).
|
|
913
|
+
let ids = deferredSeen.get(relType);
|
|
914
|
+
if (!ids) {
|
|
915
|
+
ids = new Map();
|
|
916
|
+
deferredSeen.set(relType, ids);
|
|
917
|
+
}
|
|
918
|
+
ids.delete(existing.id);
|
|
919
|
+
ids.set(existing.id, fact.lastSeenAt);
|
|
920
|
+
unchanged += 1;
|
|
921
|
+
}
|
|
332
922
|
else {
|
|
333
923
|
updateSeenStmt.run(fact.lastSeenAt, fact.orgId, fact.srcQualifiedName, fact.dstQualifiedName);
|
|
334
924
|
unchanged += 1;
|
|
335
925
|
}
|
|
336
926
|
}
|
|
337
927
|
}
|
|
928
|
+
// Deferred last_seen_at writes, grouped by value. Nothing in this call
|
|
929
|
+
// reads last_seen_at, and a later changed-attributes update of the same
|
|
930
|
+
// row dropped its entry above, so the rows end exactly as per-fact
|
|
931
|
+
// UPDATEs would leave them.
|
|
932
|
+
for (const [relType, ids] of deferredSeen) {
|
|
933
|
+
const tbl = this.edgeRelCache.get(relType);
|
|
934
|
+
const byValue = new Map();
|
|
935
|
+
for (const [id, seen] of ids) {
|
|
936
|
+
let list = byValue.get(seen);
|
|
937
|
+
if (!list) {
|
|
938
|
+
list = [];
|
|
939
|
+
byValue.set(seen, list);
|
|
940
|
+
}
|
|
941
|
+
list.push(id);
|
|
942
|
+
}
|
|
943
|
+
for (const [seen, list] of byValue) {
|
|
944
|
+
this.stmt(`UPDATE ${tbl} SET last_seen_at = @seen WHERE rowid IN (SELECT value FROM json_each(@ids))`).run({ seen, ids: JSON.stringify(list) });
|
|
945
|
+
}
|
|
946
|
+
}
|
|
947
|
+
// Prune immortal edges: for every source this batch owns, drop any
|
|
948
|
+
// OUTGOING edge it no longer emits. Scoped to (org_id, src_qname) so it's
|
|
949
|
+
// an index-served point lookup on each edge table's PK — NOT a full-table
|
|
950
|
+
// scan (which would be O(members×edges) per ingest and would read other
|
|
951
|
+
// orgs' rows). Edges produced by a different pass — MCD baseline / the
|
|
952
|
+
// reflection walker (attributes.source) and the arity resolver
|
|
953
|
+
// (attributes.resolvedBy) — are NOT touched: the parser batch never
|
|
954
|
+
// contains them, and the pre-fan-out MCD pass never re-adds them, so
|
|
955
|
+
// pruning them would silently destroy long-tail dependency coverage.
|
|
956
|
+
// Inbound edges (dst = source) are left intact regardless.
|
|
957
|
+
// Candidates are the pre-write snapshot read above (see there).
|
|
958
|
+
for (const { org, relType, r } of priorRows) {
|
|
959
|
+
const k = `${org}${RSEP}${nocaseKey(r.s)}${RSEP}${nocaseKey(r.d)}${RSEP}${relType}`;
|
|
960
|
+
if (batchEdgeKeys.has(k))
|
|
961
|
+
continue; // re-emitted this batch — keep
|
|
962
|
+
let attrs = {};
|
|
963
|
+
try {
|
|
964
|
+
attrs = r.a ? JSON.parse(r.a) : {};
|
|
965
|
+
}
|
|
966
|
+
catch {
|
|
967
|
+
attrs = {};
|
|
968
|
+
}
|
|
969
|
+
// Foreign-pass edge (MCD/reflection/arity) — not this parser's to prune.
|
|
970
|
+
if (attrs.source != null || attrs.resolvedBy != null)
|
|
971
|
+
continue;
|
|
972
|
+
const tbl = this.edgeRelCache.get(relType);
|
|
973
|
+
this.stmt(`DELETE FROM ${tbl} WHERE org_id = ? AND src_qname = ? AND dst_qname = ?`).run(org, r.s, r.d);
|
|
974
|
+
deleted += 1;
|
|
975
|
+
}
|
|
338
976
|
});
|
|
339
977
|
apply();
|
|
340
|
-
return { inserted, updated, unchanged };
|
|
978
|
+
return { inserted, updated, unchanged, ...(reconcile ? { deleted } : {}) };
|
|
979
|
+
}
|
|
980
|
+
/**
|
|
981
|
+
* Every outgoing edge of `srcs` (NOCASE, like the src_qname column) in every
|
|
982
|
+
* edge table, in one statement: a UNION ALL of per-table PK range lookups
|
|
983
|
+
* with the sources passed once as a JSON array, materialized as a CTE (no
|
|
984
|
+
* IN-list variable cap, one cached statement whatever the number of
|
|
985
|
+
* sources). Measured ~1.4x faster than one IN-list query per table.
|
|
986
|
+
*/
|
|
987
|
+
priorOutgoingEdges(org, srcs) {
|
|
988
|
+
if (this.edgeRelCache.size === 0 || srcs.length === 0)
|
|
989
|
+
return [];
|
|
990
|
+
let q = this.priorEdgesQuery;
|
|
991
|
+
if (!q || q.map !== this.edgeRelCache || q.size !== this.edgeRelCache.size) {
|
|
992
|
+
const tables = [...this.edgeRelCache];
|
|
993
|
+
const sqls = chunks(tables.map(([, tbl], i) => `SELECT ${i} AS t, rowid AS id, src_qname AS s, dst_qname AS d, attributes AS a FROM ${tbl} WHERE org_id = @org AND src_qname IN q`), MAX_UNION_ARMS).map((arms) => `WITH q(v) AS MATERIALIZED (SELECT value FROM json_each(@srcs)) ${arms.join(" UNION ALL ")}`);
|
|
994
|
+
q = { map: this.edgeRelCache, size: tables.length, rels: tables.map(([r]) => r), sqls };
|
|
995
|
+
this.priorEdgesQuery = q;
|
|
996
|
+
}
|
|
997
|
+
const rels = q.rels;
|
|
998
|
+
const params = { org, srcs: JSON.stringify(srcs) };
|
|
999
|
+
const out = [];
|
|
1000
|
+
for (const sql of q.sqls) {
|
|
1001
|
+
for (const r of this.stmt(sql).all(params)) {
|
|
1002
|
+
out.push({ relType: rels[r.t], r });
|
|
1003
|
+
}
|
|
1004
|
+
}
|
|
1005
|
+
return out;
|
|
341
1006
|
}
|
|
342
1007
|
getNode(orgId, qname) {
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
.get(orgId, qname);
|
|
1008
|
+
this.syncCaches();
|
|
1009
|
+
const idx = this.stmt("SELECT label FROM _sfgraph_node_index WHERE org_id = ? AND qualified_name = ?").get(orgId, qname);
|
|
346
1010
|
if (!idx)
|
|
347
1011
|
return null;
|
|
348
1012
|
const tbl = this.nodeLabelCache.get(idx.label);
|
|
349
1013
|
if (!tbl)
|
|
350
1014
|
return null;
|
|
351
|
-
const row = this.
|
|
352
|
-
.prepare(`SELECT org_id, qualified_name, attributes, source_hash, first_seen_at, last_seen_at, last_modified_at FROM ${tbl} WHERE org_id = ? AND qualified_name = ?`)
|
|
353
|
-
.get(orgId, qname);
|
|
1015
|
+
const row = this.stmt(`SELECT org_id, qualified_name, attributes, source_hash, first_seen_at, last_seen_at, last_modified_at FROM ${tbl} WHERE org_id = ? AND qualified_name = ?`).get(orgId, qname);
|
|
354
1016
|
if (!row)
|
|
355
1017
|
return null;
|
|
356
1018
|
return rowToNode(row, idx.label);
|
|
357
1019
|
}
|
|
358
1020
|
listNodesByLabel(orgId, label, limit) {
|
|
359
1021
|
validateLabel(label);
|
|
1022
|
+
this.syncCaches();
|
|
360
1023
|
const tbl = this.nodeLabelCache.get(label);
|
|
361
1024
|
if (!tbl)
|
|
362
1025
|
return [];
|
|
@@ -364,40 +1027,95 @@ export class SqliteGraphStore {
|
|
|
364
1027
|
const params = [orgId];
|
|
365
1028
|
if (typeof limit === "number")
|
|
366
1029
|
params.push(limit);
|
|
367
|
-
const rows = this.
|
|
1030
|
+
const rows = this.stmt(sql).all(...params);
|
|
368
1031
|
return rows.map((r) => rowToNode(r, label));
|
|
369
1032
|
}
|
|
1033
|
+
/**
|
|
1034
|
+
* listNodesByLabel's qualified names, without the attributes JSON (~6x
|
|
1035
|
+
* cheaper on attribute-heavy labels). `ORDER BY rowid` pins the order
|
|
1036
|
+
* listNodesByLabel gets from its `<tbl>_org` index scan: selecting only
|
|
1037
|
+
* qualified_name would otherwise make the covering PK index the cheapest
|
|
1038
|
+
* plan, which yields NOCASE name order (and a different LIMIT subset).
|
|
1039
|
+
*/
|
|
1040
|
+
listQnamesByLabel(orgId, label, limit) {
|
|
1041
|
+
validateLabel(label);
|
|
1042
|
+
this.syncCaches();
|
|
1043
|
+
const tbl = this.nodeLabelCache.get(label);
|
|
1044
|
+
if (!tbl)
|
|
1045
|
+
return [];
|
|
1046
|
+
const sql = `SELECT qualified_name FROM ${tbl} WHERE org_id = ? ORDER BY rowid${typeof limit === "number" ? " LIMIT ?" : ""}`;
|
|
1047
|
+
const params = [orgId];
|
|
1048
|
+
if (typeof limit === "number")
|
|
1049
|
+
params.push(limit);
|
|
1050
|
+
const rows = this.stmt(sql).all(...params);
|
|
1051
|
+
return rows.map((r) => asQualifiedName(r.qualified_name));
|
|
1052
|
+
}
|
|
1053
|
+
/**
|
|
1054
|
+
* Every edge table's rows with `col = @q`, as ONE statement: a UNION ALL of
|
|
1055
|
+
* the per-table queries listEdgesFrom / listEdgesTo used to run one by one,
|
|
1056
|
+
* in edgeRelCache order. SQLite runs a UNION ALL's arms in order, each with
|
|
1057
|
+
* the plan it gets on its own, so the rows (and their order) are exactly
|
|
1058
|
+
* the concatenation of the per-table results. One statement step per
|
|
1059
|
+
* traversal node instead of one per edge table (~2x faster traces).
|
|
1060
|
+
*/
|
|
1061
|
+
edgesAllTables(col, orgId, q) {
|
|
1062
|
+
let u = this.unionEdgeQueries[col];
|
|
1063
|
+
if (!u || u.map !== this.edgeRelCache || u.size !== this.edgeRelCache.size) {
|
|
1064
|
+
const tables = [...this.edgeRelCache];
|
|
1065
|
+
u = {
|
|
1066
|
+
map: this.edgeRelCache,
|
|
1067
|
+
size: tables.length,
|
|
1068
|
+
rels: tables.map(([r]) => r),
|
|
1069
|
+
// Consecutive groups of tables, run in order (SQLite caps compound arms).
|
|
1070
|
+
sqls: chunks(tables.map(([, tbl], i) => `SELECT ${i} AS t, org_id, src_qname, dst_qname, attributes, first_seen_at, last_seen_at FROM ${tbl} WHERE org_id = @org AND ${col} = @q`), MAX_UNION_ARMS).map((arms) => arms.join(" UNION ALL ")),
|
|
1071
|
+
};
|
|
1072
|
+
this.unionEdgeQueries[col] = u;
|
|
1073
|
+
}
|
|
1074
|
+
const rels = u.rels;
|
|
1075
|
+
const out = [];
|
|
1076
|
+
for (const sql of u.sqls) {
|
|
1077
|
+
const rows = this.stmt(sql).all({ org: orgId, q });
|
|
1078
|
+
for (const r of rows)
|
|
1079
|
+
out.push(rowToEdge(r, rels[r.t]));
|
|
1080
|
+
}
|
|
1081
|
+
return out;
|
|
1082
|
+
}
|
|
370
1083
|
listEdgesFrom(orgId, src, relType) {
|
|
1084
|
+
this.syncCaches();
|
|
1085
|
+
if (!relType)
|
|
1086
|
+
return this.edgesAllTables("src_qname", orgId, src);
|
|
371
1087
|
const out = [];
|
|
372
|
-
const types =
|
|
1088
|
+
const types = [relType];
|
|
373
1089
|
for (const t of types) {
|
|
374
1090
|
const tbl = this.edgeRelCache.get(t);
|
|
375
1091
|
if (!tbl)
|
|
376
1092
|
continue;
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
.all(orgId, src);
|
|
1093
|
+
// perf-05: cached per table; traversals call this once per BFS node.
|
|
1094
|
+
const rows = this.stmt(`SELECT org_id, src_qname, dst_qname, attributes, first_seen_at, last_seen_at FROM ${tbl} WHERE org_id = ? AND src_qname = ?`).all(orgId, src);
|
|
380
1095
|
for (const r of rows)
|
|
381
1096
|
out.push(rowToEdge(r, t));
|
|
382
1097
|
}
|
|
383
1098
|
return out;
|
|
384
1099
|
}
|
|
385
1100
|
listEdgesTo(orgId, dst, relType) {
|
|
1101
|
+
this.syncCaches();
|
|
1102
|
+
if (!relType)
|
|
1103
|
+
return this.edgesAllTables("dst_qname", orgId, dst);
|
|
386
1104
|
const out = [];
|
|
387
|
-
const types =
|
|
1105
|
+
const types = [relType];
|
|
388
1106
|
for (const t of types) {
|
|
389
1107
|
const tbl = this.edgeRelCache.get(t);
|
|
390
1108
|
if (!tbl)
|
|
391
1109
|
continue;
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
.all(orgId, dst);
|
|
1110
|
+
// perf-05: cached per table; traversals call this once per BFS node.
|
|
1111
|
+
const rows = this.stmt(`SELECT org_id, src_qname, dst_qname, attributes, first_seen_at, last_seen_at FROM ${tbl} WHERE org_id = ? AND dst_qname = ?`).all(orgId, dst);
|
|
395
1112
|
for (const r of rows)
|
|
396
1113
|
out.push(rowToEdge(r, t));
|
|
397
1114
|
}
|
|
398
1115
|
return out;
|
|
399
1116
|
}
|
|
400
1117
|
listEdgesByDstLike(orgId, pattern, relType, limit) {
|
|
1118
|
+
this.syncCaches();
|
|
401
1119
|
const out = [];
|
|
402
1120
|
const types = relType ? [relType] : Array.from(this.edgeRelCache.keys());
|
|
403
1121
|
const cap = limit && limit > 0 ? limit : Number.POSITIVE_INFINITY;
|
|
@@ -407,17 +1125,16 @@ export class SqliteGraphStore {
|
|
|
407
1125
|
const tbl = this.edgeRelCache.get(t);
|
|
408
1126
|
if (!tbl)
|
|
409
1127
|
continue;
|
|
1128
|
+
// LIMIT -1 is SQLite's "no limit": one cached statement per table.
|
|
410
1129
|
const remaining = Number.isFinite(cap) ? cap - out.length : -1;
|
|
411
|
-
const
|
|
412
|
-
? `SELECT org_id, src_qname, dst_qname, attributes, first_seen_at, last_seen_at FROM ${tbl} WHERE org_id = ? AND dst_qname LIKE ? LIMIT ${remaining}`
|
|
413
|
-
: `SELECT org_id, src_qname, dst_qname, attributes, first_seen_at, last_seen_at FROM ${tbl} WHERE org_id = ? AND dst_qname LIKE ?`;
|
|
414
|
-
const rows = this.db.prepare(sql).all(orgId, pattern);
|
|
1130
|
+
const rows = this.stmt(`SELECT org_id, src_qname, dst_qname, attributes, first_seen_at, last_seen_at FROM ${tbl} WHERE org_id = ? AND dst_qname LIKE ? LIMIT ?`).all(orgId, pattern, remaining);
|
|
415
1131
|
for (const r of rows)
|
|
416
1132
|
out.push(rowToEdge(r, t));
|
|
417
1133
|
}
|
|
418
1134
|
return out;
|
|
419
1135
|
}
|
|
420
1136
|
deleteEdge(orgId, src, dst, relType) {
|
|
1137
|
+
this.syncCaches();
|
|
421
1138
|
const tbl = this.edgeRelCache.get(relType);
|
|
422
1139
|
if (!tbl)
|
|
423
1140
|
return;
|
|
@@ -433,6 +1150,7 @@ export class SqliteGraphStore {
|
|
|
433
1150
|
* field refs whose CustomObject wasn't ingested).
|
|
434
1151
|
*/
|
|
435
1152
|
listDanglingEdges(orgId, limit) {
|
|
1153
|
+
this.syncCaches();
|
|
436
1154
|
const out = [];
|
|
437
1155
|
const cap = limit && limit > 0 ? limit : Number.POSITIVE_INFINITY;
|
|
438
1156
|
for (const [relType, tbl] of this.edgeRelCache) {
|
|
@@ -453,12 +1171,41 @@ export class SqliteGraphStore {
|
|
|
453
1171
|
}
|
|
454
1172
|
return out;
|
|
455
1173
|
}
|
|
1174
|
+
/**
|
|
1175
|
+
* Streaming form of `listDanglingEdges`: same SQL, tables and order, one
|
|
1176
|
+
* row at a time and without parsing attributes, so an org with ~1M
|
|
1177
|
+
* dangling edges never holds them all at once.
|
|
1178
|
+
*/
|
|
1179
|
+
*iterateDanglingEdges(orgId, limit) {
|
|
1180
|
+
this.syncCaches();
|
|
1181
|
+
const cap = limit && limit > 0 ? limit : Number.POSITIVE_INFINITY;
|
|
1182
|
+
let seen = 0;
|
|
1183
|
+
for (const [relType, tbl] of [...this.edgeRelCache]) {
|
|
1184
|
+
if (seen >= cap)
|
|
1185
|
+
return;
|
|
1186
|
+
const remaining = Number.isFinite(cap) ? cap - seen : -1;
|
|
1187
|
+
const limitClause = remaining > 0 ? `LIMIT ${remaining}` : "";
|
|
1188
|
+
const rows = this.db
|
|
1189
|
+
.prepare(`SELECT e.src_qname, e.dst_qname, e.attributes
|
|
1190
|
+
FROM ${tbl} e
|
|
1191
|
+
LEFT JOIN _sfgraph_node_index n
|
|
1192
|
+
ON n.org_id = e.org_id AND n.qualified_name = e.dst_qname
|
|
1193
|
+
WHERE e.org_id = ? AND n.qualified_name IS NULL
|
|
1194
|
+
${limitClause}`)
|
|
1195
|
+
.iterate(orgId);
|
|
1196
|
+
for (const r of rows) {
|
|
1197
|
+
seen++;
|
|
1198
|
+
yield { relType, src: r.src_qname, dst: r.dst_qname, attributes: r.attributes };
|
|
1199
|
+
}
|
|
1200
|
+
}
|
|
1201
|
+
}
|
|
456
1202
|
/**
|
|
457
1203
|
* Return every qualified name persisted for `orgId` across all label tables.
|
|
458
1204
|
* Used by full-sync deletion detection to compute the difference between
|
|
459
1205
|
* what was previously known and what the current sync touched.
|
|
460
1206
|
*/
|
|
461
1207
|
listAllQnames(orgId) {
|
|
1208
|
+
this.syncCaches();
|
|
462
1209
|
const out = [];
|
|
463
1210
|
for (const tbl of this.nodeLabelCache.values()) {
|
|
464
1211
|
const rows = this.db
|
|
@@ -475,9 +1222,11 @@ export class SqliteGraphStore {
|
|
|
475
1222
|
* exposing the private nodeLabelCache. Order is undefined.
|
|
476
1223
|
*/
|
|
477
1224
|
listAllLabels() {
|
|
1225
|
+
this.syncCaches();
|
|
478
1226
|
return [...this.nodeLabelCache.keys()];
|
|
479
1227
|
}
|
|
480
1228
|
countNodes(orgId) {
|
|
1229
|
+
this.syncCaches();
|
|
481
1230
|
let total = 0;
|
|
482
1231
|
for (const tbl of this.nodeLabelCache.values()) {
|
|
483
1232
|
const row = this.db.prepare(`SELECT COUNT(*) AS c FROM ${tbl} WHERE org_id = ?`).get(orgId);
|
|
@@ -485,7 +1234,25 @@ export class SqliteGraphStore {
|
|
|
485
1234
|
}
|
|
486
1235
|
return total;
|
|
487
1236
|
}
|
|
1237
|
+
/**
|
|
1238
|
+
* Count nodes of `label` for `orgId`. Returns 0 if the label table has
|
|
1239
|
+
* not been created yet. Used by the W1.5-07 detect-deletions guard.
|
|
1240
|
+
*
|
|
1241
|
+
* Note: a similar helper `countNodes(orgId)` already exists for whole-org
|
|
1242
|
+
* counts, but it sums across all labels. The guard needs per-label
|
|
1243
|
+
* precision to compute drop ratios per deletion-target, so a dedicated
|
|
1244
|
+
* helper is added rather than overloading countNodes.
|
|
1245
|
+
*/
|
|
1246
|
+
countNodesByLabel(orgId, label) {
|
|
1247
|
+
this.syncCaches();
|
|
1248
|
+
const tbl = this.nodeLabelCache.get(label);
|
|
1249
|
+
if (!tbl)
|
|
1250
|
+
return 0;
|
|
1251
|
+
const row = this.db.prepare(`SELECT COUNT(*) AS c FROM ${tbl} WHERE org_id = ?`).get(orgId);
|
|
1252
|
+
return row?.c ?? 0;
|
|
1253
|
+
}
|
|
488
1254
|
countEdges(orgId) {
|
|
1255
|
+
this.syncCaches();
|
|
489
1256
|
let total = 0;
|
|
490
1257
|
for (const tbl of this.edgeRelCache.values()) {
|
|
491
1258
|
const row = this.db.prepare(`SELECT COUNT(*) AS c FROM ${tbl} WHERE org_id = ?`).get(orgId);
|
|
@@ -497,26 +1264,20 @@ export class SqliteGraphStore {
|
|
|
497
1264
|
return this.db.transaction(fn)();
|
|
498
1265
|
}
|
|
499
1266
|
upsertSnippet(rec) {
|
|
500
|
-
const existing = this.
|
|
501
|
-
.prepare("SELECT source_hash FROM _sfgraph_snippets WHERE org_id = ? AND qualified_name = ?")
|
|
502
|
-
.get(rec.orgId, rec.qualifiedName);
|
|
1267
|
+
const existing = this.stmt("SELECT source_hash FROM _sfgraph_snippets WHERE org_id = ? AND qualified_name = ?").get(rec.orgId, rec.qualifiedName);
|
|
503
1268
|
if (existing && existing.source_hash === rec.sourceHash) {
|
|
504
1269
|
return { inserted: false, updated: false, unchanged: true };
|
|
505
1270
|
}
|
|
506
1271
|
if (existing) {
|
|
507
|
-
this.
|
|
508
|
-
.prepare(`UPDATE _sfgraph_snippets
|
|
1272
|
+
this.stmt(`UPDATE _sfgraph_snippets
|
|
509
1273
|
SET source_format = ?, source_text = ?, start_line = ?, end_line = ?,
|
|
510
1274
|
source_hash = ?, llm_explanation = NULL, explained_at = NULL
|
|
511
|
-
WHERE org_id = ? AND qualified_name = ?`)
|
|
512
|
-
.run(rec.sourceFormat, rec.sourceText, rec.startLine ?? null, rec.endLine ?? null, rec.sourceHash, rec.orgId, rec.qualifiedName);
|
|
1275
|
+
WHERE org_id = ? AND qualified_name = ?`).run(rec.sourceFormat, rec.sourceText, rec.startLine ?? null, rec.endLine ?? null, rec.sourceHash, rec.orgId, rec.qualifiedName);
|
|
513
1276
|
return { inserted: false, updated: true, unchanged: false };
|
|
514
1277
|
}
|
|
515
|
-
this.
|
|
516
|
-
.prepare(`INSERT INTO _sfgraph_snippets
|
|
1278
|
+
this.stmt(`INSERT INTO _sfgraph_snippets
|
|
517
1279
|
(org_id, qualified_name, source_format, source_text, start_line, end_line, source_hash, llm_explanation, explained_at)
|
|
518
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
519
|
-
.run(rec.orgId, rec.qualifiedName, rec.sourceFormat, rec.sourceText, rec.startLine ?? null, rec.endLine ?? null, rec.sourceHash, rec.llmExplanation ?? null, rec.explainedAt ?? null);
|
|
1280
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(rec.orgId, rec.qualifiedName, rec.sourceFormat, rec.sourceText, rec.startLine ?? null, rec.endLine ?? null, rec.sourceHash, rec.llmExplanation ?? null, rec.explainedAt ?? null);
|
|
520
1281
|
return { inserted: true, updated: false, unchanged: false };
|
|
521
1282
|
}
|
|
522
1283
|
getSnippet(orgId, qname) {
|
|
@@ -576,15 +1337,941 @@ export class SqliteGraphStore {
|
|
|
576
1337
|
return rec;
|
|
577
1338
|
});
|
|
578
1339
|
}
|
|
1340
|
+
/**
|
|
1341
|
+
* Replace the FTS doc for (org, qname). The old row is found through the
|
|
1342
|
+
* indexed _sfgraph_node_fts_rowid map and deleted by rowid; a WHERE on the
|
|
1343
|
+
* FTS5 UNINDEXED key columns would scan the whole table (O(n²) per ingest).
|
|
1344
|
+
* Keys compare BINARY, exactly like the old column equality, and the insert
|
|
1345
|
+
* sequence (hence rowids and bm25 tie order) is unchanged.
|
|
1346
|
+
*/
|
|
1347
|
+
upsertNodeFts(orgId, qname, label, body) {
|
|
1348
|
+
try {
|
|
1349
|
+
if (!this.ftsUpsertTx) {
|
|
1350
|
+
this.ftsUpsertTx = this.db.transaction((org, q, lbl, text) => {
|
|
1351
|
+
const bulk = this.bulk;
|
|
1352
|
+
const key = `${org}${KSEP}${q}`;
|
|
1353
|
+
if (bulk && bulk.freshOrg === org && !bulk.ftsKeys.has(key)) {
|
|
1354
|
+
// Fresh org, first write of this key in the window: nothing to delete.
|
|
1355
|
+
bulk.stats.ftsDeleteSkipped += 1;
|
|
1356
|
+
}
|
|
1357
|
+
else {
|
|
1358
|
+
this.deleteNodeFts(org, q);
|
|
1359
|
+
}
|
|
1360
|
+
const info = this.stmt("INSERT INTO _sfgraph_node_fts(qualified_name, org_id, label, body) VALUES (?, ?, ?, ?)").run(q, org, lbl, text);
|
|
1361
|
+
this.stmt(`INSERT INTO _sfgraph_node_fts_rowid(org_id, qualified_name, fts_rowid) VALUES (?, ?, ?)
|
|
1362
|
+
ON CONFLICT(org_id, qualified_name) DO UPDATE SET fts_rowid = excluded.fts_rowid`).run(org, q, info.lastInsertRowid);
|
|
1363
|
+
if (bulk && bulk.freshOrg === org)
|
|
1364
|
+
bulk.ftsKeys.add(key);
|
|
1365
|
+
});
|
|
1366
|
+
}
|
|
1367
|
+
this.ftsUpsertTx(String(orgId), String(qname), label, body);
|
|
1368
|
+
}
|
|
1369
|
+
catch {
|
|
1370
|
+
/* FTS5 unavailable — keyword leg degrades to empty. */
|
|
1371
|
+
}
|
|
1372
|
+
}
|
|
1373
|
+
/** Delete (org, qname)'s FTS doc by rowid via the side table. Best-effort when FTS5 is absent. */
|
|
1374
|
+
deleteNodeFts(orgId, qname) {
|
|
1375
|
+
let row;
|
|
1376
|
+
try {
|
|
1377
|
+
row = this.stmt("SELECT fts_rowid AS r FROM _sfgraph_node_fts_rowid WHERE org_id = ? AND qualified_name = ?").get(orgId, qname);
|
|
1378
|
+
}
|
|
1379
|
+
catch {
|
|
1380
|
+
return; // pre-migration / FTS-less store
|
|
1381
|
+
}
|
|
1382
|
+
if (!row)
|
|
1383
|
+
return;
|
|
1384
|
+
try {
|
|
1385
|
+
this.stmt("DELETE FROM _sfgraph_node_fts WHERE rowid = ?").run(row.r);
|
|
1386
|
+
}
|
|
1387
|
+
catch {
|
|
1388
|
+
/* FTS5 unavailable */
|
|
1389
|
+
}
|
|
1390
|
+
this.stmt("DELETE FROM _sfgraph_node_fts_rowid WHERE org_id = ? AND qualified_name = ?").run(orgId, qname);
|
|
1391
|
+
}
|
|
1392
|
+
// ─── Task 10: bulk-load window for rebuilds ───────────────────────────────
|
|
1393
|
+
/**
|
|
1394
|
+
* Self-heal after a process died inside a bulk window (crash, SIGKILL, OOM,
|
|
1395
|
+
* native checkpoint abort): the DROP INDEXes were durable but endBulk never
|
|
1396
|
+
* rebuilt them. Recreates every missing canonical `_org` / `_rev` index and
|
|
1397
|
+
* clears the marker. Skipped unless the marker is ours or its owner is
|
|
1398
|
+
* stale (same host, dead pid): another live process's window, or any
|
|
1399
|
+
* window opened on another host, is legitimately open. Cheap when nothing is missing:
|
|
1400
|
+
* one sqlite_master read. Best-effort — a busy DB must not fail init().
|
|
1401
|
+
*/
|
|
1402
|
+
/** Task 11c: markers written before the host column get it added (null = this host). */
|
|
1403
|
+
ensureBulkMarkerHost() {
|
|
1404
|
+
const cols = this.db.prepare("PRAGMA table_info(_sfgraph_bulk_window)").all();
|
|
1405
|
+
if (!cols.some((c) => c.name === "host")) {
|
|
1406
|
+
try {
|
|
1407
|
+
this.db.exec("ALTER TABLE _sfgraph_bulk_window ADD COLUMN host TEXT");
|
|
1408
|
+
}
|
|
1409
|
+
catch {
|
|
1410
|
+
/* a concurrent opener added it */
|
|
1411
|
+
}
|
|
1412
|
+
}
|
|
1413
|
+
}
|
|
1414
|
+
healBulkWindow() {
|
|
1415
|
+
try {
|
|
1416
|
+
const hasMarkerTable = this.db
|
|
1417
|
+
.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = '_sfgraph_bulk_window'")
|
|
1418
|
+
.get() !== undefined;
|
|
1419
|
+
if (hasMarkerTable)
|
|
1420
|
+
this.ensureBulkMarkerHost();
|
|
1421
|
+
const marker = hasMarkerTable
|
|
1422
|
+
? this.db
|
|
1423
|
+
.prepare("SELECT pid, host, started_at FROM _sfgraph_bulk_window WHERE id = 1")
|
|
1424
|
+
.get()
|
|
1425
|
+
: undefined;
|
|
1426
|
+
// Heal only our own window or one whose owner is provably gone: same
|
|
1427
|
+
// host, dead pid. A window opened on another host is never touched.
|
|
1428
|
+
if (marker) {
|
|
1429
|
+
const ours = marker.pid === process.pid && isThisHost(marker.host);
|
|
1430
|
+
if (!ours && !ownerIsStale({ ...marker, takenAtMs: marker.started_at }))
|
|
1431
|
+
return;
|
|
1432
|
+
}
|
|
1433
|
+
const objects = this.db
|
|
1434
|
+
.prepare("SELECT type, name FROM sqlite_master WHERE type IN ('table', 'index')")
|
|
1435
|
+
.all();
|
|
1436
|
+
const tables = new Set(objects.filter((o) => o.type === "table").map((o) => o.name));
|
|
1437
|
+
const idx = new Set(objects.filter((o) => o.type === "index").map((o) => o.name));
|
|
1438
|
+
const missing = [];
|
|
1439
|
+
for (const tbl of this.nodeLabelCache.values())
|
|
1440
|
+
if (tables.has(tbl) && !idx.has(`${tbl}_org`))
|
|
1441
|
+
missing.push(nodeIndexDdl(tbl));
|
|
1442
|
+
for (const tbl of this.edgeRelCache.values())
|
|
1443
|
+
if (tables.has(tbl) && !idx.has(`${tbl}_rev`))
|
|
1444
|
+
missing.push(edgeIndexDdl(tbl));
|
|
1445
|
+
if (missing.length === 0 && !marker)
|
|
1446
|
+
return;
|
|
1447
|
+
this.db.transaction(() => {
|
|
1448
|
+
for (const sql of missing)
|
|
1449
|
+
this.db.exec(sql);
|
|
1450
|
+
if (marker)
|
|
1451
|
+
this.db.prepare("DELETE FROM _sfgraph_bulk_window WHERE id = 1").run();
|
|
1452
|
+
})();
|
|
1453
|
+
}
|
|
1454
|
+
catch {
|
|
1455
|
+
/* best-effort: e.g. SQLITE_BUSY; retried on the next init() */
|
|
1456
|
+
}
|
|
1457
|
+
}
|
|
1458
|
+
// ─── Task 11a: embedding status ───────────────────────────────────────────
|
|
1459
|
+
/** Mark the org's vectors as still being written (or done). Best-effort on
|
|
1460
|
+
* a pre-v12 store. */
|
|
1461
|
+
setEmbeddingsPending(orgId, pending) {
|
|
1462
|
+
try {
|
|
1463
|
+
this.stmt("UPDATE _sfgraph_orgs SET embeddings_pending = ? WHERE id = ?").run(pending ? 1 : 0, orgId);
|
|
1464
|
+
}
|
|
1465
|
+
catch {
|
|
1466
|
+
/* pre-v12 store */
|
|
1467
|
+
}
|
|
1468
|
+
}
|
|
1469
|
+
/**
|
|
1470
|
+
* Whether the org's vectors are still being written, and how many texts it
|
|
1471
|
+
* embeds in total (its FTS docs: every embedded text is also an FTS doc).
|
|
1472
|
+
* Null for an unknown org or a store without the v12 column.
|
|
1473
|
+
*/
|
|
1474
|
+
getEmbeddingStatus(orgId) {
|
|
1475
|
+
let row;
|
|
1476
|
+
try {
|
|
1477
|
+
row = this.stmt("SELECT embeddings_pending AS p FROM _sfgraph_orgs WHERE id = ?").get(orgId);
|
|
1478
|
+
}
|
|
1479
|
+
catch {
|
|
1480
|
+
return null;
|
|
1481
|
+
}
|
|
1482
|
+
if (!row)
|
|
1483
|
+
return null;
|
|
1484
|
+
let total = 0;
|
|
1485
|
+
try {
|
|
1486
|
+
total = this.stmt("SELECT COUNT(*) AS c FROM _sfgraph_node_fts_rowid WHERE org_id = ?").get(orgId).c;
|
|
1487
|
+
}
|
|
1488
|
+
catch {
|
|
1489
|
+
/* FTS-less store */
|
|
1490
|
+
}
|
|
1491
|
+
return { pending: row.p === 1, total };
|
|
1492
|
+
}
|
|
1493
|
+
/** Every FTS doc of the org (the exact embed text per qname), for the
|
|
1494
|
+
* embedding backfill. Empty when FTS5 is unavailable. Streams in keyset
|
|
1495
|
+
* pages of `pageSize` rows, so at most one page of bodies is held at once.
|
|
1496
|
+
* Not `.iterate()`: the caller writes vectors between rows (backpressure
|
|
1497
|
+
* can force a dispatch), and a better-sqlite3 connection with an open
|
|
1498
|
+
* iterator rejects writes ("connection is busy"). */
|
|
1499
|
+
*listFtsDocs(orgId, pageSize = 1000) {
|
|
1500
|
+
// Keyset on the (org_id, qualified_name) primary key: each page is an
|
|
1501
|
+
// index range scan, not a re-sort of the whole org.
|
|
1502
|
+
let after = "";
|
|
1503
|
+
for (;;) {
|
|
1504
|
+
let rows;
|
|
1505
|
+
try {
|
|
1506
|
+
rows = this.stmt(`SELECT r.qualified_name AS qname, f.label AS label, f.body AS body
|
|
1507
|
+
FROM _sfgraph_node_fts_rowid r
|
|
1508
|
+
JOIN _sfgraph_node_fts f ON f.rowid = r.fts_rowid
|
|
1509
|
+
WHERE r.org_id = ? AND r.qualified_name > ?
|
|
1510
|
+
ORDER BY r.qualified_name
|
|
1511
|
+
LIMIT ?`).all(orgId, after, pageSize);
|
|
1512
|
+
}
|
|
1513
|
+
catch {
|
|
1514
|
+
return;
|
|
1515
|
+
}
|
|
1516
|
+
yield* rows;
|
|
1517
|
+
if (rows.length < pageSize)
|
|
1518
|
+
return;
|
|
1519
|
+
after = rows[rows.length - 1].qname;
|
|
1520
|
+
}
|
|
1521
|
+
}
|
|
1522
|
+
/** True when `orgId` has no node, edge or FTS rows. Index probes only (LIMIT 1). */
|
|
1523
|
+
isOrgEmpty(orgId) {
|
|
1524
|
+
this.syncCaches();
|
|
1525
|
+
const probe = (tbl) => this.db.prepare(`SELECT 1 FROM ${tbl} WHERE org_id = ? LIMIT 1`).get(orgId) !== undefined;
|
|
1526
|
+
for (const tbl of this.nodeLabelCache.values())
|
|
1527
|
+
if (probe(tbl))
|
|
1528
|
+
return false;
|
|
1529
|
+
for (const tbl of this.edgeRelCache.values())
|
|
1530
|
+
if (probe(tbl))
|
|
1531
|
+
return false;
|
|
1532
|
+
if (probe("_sfgraph_node_index"))
|
|
1533
|
+
return false;
|
|
1534
|
+
try {
|
|
1535
|
+
if (probe("_sfgraph_node_fts_rowid"))
|
|
1536
|
+
return false;
|
|
1537
|
+
}
|
|
1538
|
+
catch {
|
|
1539
|
+
/* table absent */
|
|
1540
|
+
}
|
|
1541
|
+
return true;
|
|
1542
|
+
}
|
|
1543
|
+
/**
|
|
1544
|
+
* Open a bulk-load window for a rebuild: drop the graph tables' secondary
|
|
1545
|
+
* indexes (rebuilt in one pass at endBulk), set journal_mode=WAL,
|
|
1546
|
+
* synchronous=OFF, temp_store=MEMORY, and hold one write transaction per
|
|
1547
|
+
* BULK_TXN_MEMBERS members (advanced by bulkMemberDone). Not re-entrant.
|
|
1548
|
+
*/
|
|
1549
|
+
beginBulk(opts = {}) {
|
|
1550
|
+
if (this.bulk)
|
|
1551
|
+
throw new StorageError("beginBulk: a bulk window is already open");
|
|
1552
|
+
if (this.batch)
|
|
1553
|
+
throw new StorageError("beginBulk: a batch window is open");
|
|
1554
|
+
if (this.db.inTransaction)
|
|
1555
|
+
throw new StorageError("beginBulk: called inside a transaction");
|
|
1556
|
+
this.syncCaches();
|
|
1557
|
+
const fresh = opts.freshOrgId !== undefined && this.isOrgEmpty(opts.freshOrgId)
|
|
1558
|
+
? String(opts.freshOrgId)
|
|
1559
|
+
: null;
|
|
1560
|
+
const batchTxn = opts.batchTransactions !== false;
|
|
1561
|
+
const graphTables = new Set([...this.nodeLabelCache.values(), ...this.edgeRelCache.values()]);
|
|
1562
|
+
// The _sfg_* tables are shared by every org in the file. A fresh org next
|
|
1563
|
+
// to other orgs' rows keeps the existing indexes (their readers, and an
|
|
1564
|
+
// endBulk re-index of every org's rows, would otherwise pay); only tables
|
|
1565
|
+
// created inside the window defer their indexes.
|
|
1566
|
+
const otherOrgsHaveRows = fresh !== null &&
|
|
1567
|
+
[...graphTables].some((t) => this.db.prepare(`SELECT 1 FROM ${t} LIMIT 1`).get() !== undefined);
|
|
1568
|
+
const existing = otherOrgsHaveRows
|
|
1569
|
+
? []
|
|
1570
|
+
: this.db
|
|
1571
|
+
.prepare("SELECT name, tbl_name AS tbl, sql FROM sqlite_master WHERE type = 'index' AND sql IS NOT NULL")
|
|
1572
|
+
.all().filter((i) => graphTables.has(i.tbl));
|
|
1573
|
+
// Durable marker first (autocommit), so a crash after any DROP is healed.
|
|
1574
|
+
this.db.exec(BULK_MARKER_DDL);
|
|
1575
|
+
this.ensureBulkMarkerHost();
|
|
1576
|
+
this.db
|
|
1577
|
+
.prepare("INSERT OR REPLACE INTO _sfgraph_bulk_window(id, pid, started_at, host) VALUES (1, ?, ?, ?)")
|
|
1578
|
+
.run(process.pid, Date.now(), thisHost());
|
|
1579
|
+
// Register the window before touching anything, so a failure part-way
|
|
1580
|
+
// through is undone by endBulk (indexes rebuilt, synchronous restored).
|
|
1581
|
+
this.bulk = {
|
|
1582
|
+
freshOrg: fresh,
|
|
1583
|
+
batchTxn,
|
|
1584
|
+
members: 0,
|
|
1585
|
+
edgeSrcs: new Set(),
|
|
1586
|
+
ftsKeys: new Set(),
|
|
1587
|
+
deferredIndexes: existing.map((i) => ({ name: i.name, table: i.tbl, sql: i.sql })),
|
|
1588
|
+
stats: {
|
|
1589
|
+
active: true,
|
|
1590
|
+
fresh: fresh !== null,
|
|
1591
|
+
members: 0,
|
|
1592
|
+
reconcileSkipped: 0,
|
|
1593
|
+
reconcileProbed: 0,
|
|
1594
|
+
ftsDeleteSkipped: 0,
|
|
1595
|
+
},
|
|
1596
|
+
};
|
|
1597
|
+
try {
|
|
1598
|
+
this.db.pragma("journal_mode = WAL");
|
|
1599
|
+
this.db.pragma("synchronous = OFF");
|
|
1600
|
+
this.db.pragma("temp_store = MEMORY");
|
|
1601
|
+
for (const i of existing)
|
|
1602
|
+
this.db.exec(`DROP INDEX IF EXISTS "${i.name.replace(/"/g, '""')}"`);
|
|
1603
|
+
if (batchTxn)
|
|
1604
|
+
this.db.exec("BEGIN");
|
|
1605
|
+
}
|
|
1606
|
+
catch (e) {
|
|
1607
|
+
try {
|
|
1608
|
+
this.endBulk();
|
|
1609
|
+
}
|
|
1610
|
+
catch {
|
|
1611
|
+
/* the original error matters more; init() heals on next open */
|
|
1612
|
+
}
|
|
1613
|
+
throw e;
|
|
1614
|
+
}
|
|
1615
|
+
}
|
|
1616
|
+
/** Mark one member written; commits and reopens the bulk transaction every BULK_TXN_MEMBERS. */
|
|
1617
|
+
bulkMemberDone() {
|
|
1618
|
+
const bulk = this.bulk;
|
|
1619
|
+
if (!bulk)
|
|
1620
|
+
return;
|
|
1621
|
+
if (bulk.batchTxn && !this.db.inTransaction) {
|
|
1622
|
+
// SQLite rolls back the whole outer BEGIN on SQLITE_FULL / IOERR / NOMEM
|
|
1623
|
+
// inside a member's savepoint: every member since the last commit is lost.
|
|
1624
|
+
throw new StorageError("bulk transaction was rolled back by SQLite; members written since the last commit were lost");
|
|
1625
|
+
}
|
|
1626
|
+
bulk.members += 1;
|
|
1627
|
+
bulk.stats.members = bulk.members;
|
|
1628
|
+
if (bulk.batchTxn && bulk.members % BULK_TXN_MEMBERS === 0 && this.db.inTransaction) {
|
|
1629
|
+
this.db.exec("COMMIT");
|
|
1630
|
+
this.db.exec("BEGIN");
|
|
1631
|
+
}
|
|
1632
|
+
}
|
|
1633
|
+
/**
|
|
1634
|
+
* Close the bulk window: commit, recreate the deferred indexes (with
|
|
1635
|
+
* temp_store=FILE, then the previous temp_store is restored), restore
|
|
1636
|
+
* synchronous=NORMAL. No-op when closed.
|
|
1637
|
+
*/
|
|
1638
|
+
endBulk() {
|
|
1639
|
+
const bulk = this.bulk;
|
|
1640
|
+
if (!bulk)
|
|
1641
|
+
return;
|
|
1642
|
+
this.bulk = null;
|
|
1643
|
+
try {
|
|
1644
|
+
if (bulk.batchTxn && this.db.inTransaction)
|
|
1645
|
+
this.db.exec("COMMIT");
|
|
1646
|
+
}
|
|
1647
|
+
finally {
|
|
1648
|
+
const build = this.db.transaction(() => {
|
|
1649
|
+
for (const i of bulk.deferredIndexes) {
|
|
1650
|
+
const hasTable = this.db
|
|
1651
|
+
.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?")
|
|
1652
|
+
.get(i.table);
|
|
1653
|
+
const hasIndex = this.db
|
|
1654
|
+
.prepare("SELECT 1 FROM sqlite_master WHERE type = 'index' AND name = ?")
|
|
1655
|
+
.get(i.name);
|
|
1656
|
+
if (hasTable && !hasIndex)
|
|
1657
|
+
this.db.exec(i.sql);
|
|
1658
|
+
}
|
|
1659
|
+
});
|
|
1660
|
+
// The index sorter spills past cache_size into temp files; with
|
|
1661
|
+
// temp_store=MEMORY those live in RAM (+~1 GB RSS on a 2.6M-edge org,
|
|
1662
|
+
// never returned by malloc). Spill to disk for the rebuild only.
|
|
1663
|
+
// SQLite refuses to change temp_store inside a transaction.
|
|
1664
|
+
const prevTempStore = this.db.inTransaction
|
|
1665
|
+
? null
|
|
1666
|
+
: Number(this.db.pragma("temp_store", { simple: true }));
|
|
1667
|
+
if (prevTempStore === null)
|
|
1668
|
+
console.warn("sfgraph: endBulk is inside a transaction (the bulk commit failed?); rebuilding indexes with the connection's temp_store (MEMORY), which can use much more RAM");
|
|
1669
|
+
try {
|
|
1670
|
+
if (prevTempStore !== null && prevTempStore !== 1)
|
|
1671
|
+
this.db.pragma("temp_store = FILE");
|
|
1672
|
+
build();
|
|
1673
|
+
this.db.prepare("DELETE FROM _sfgraph_bulk_window WHERE id = 1").run();
|
|
1674
|
+
}
|
|
1675
|
+
finally {
|
|
1676
|
+
try {
|
|
1677
|
+
if (prevTempStore !== null && prevTempStore !== 1 && !this.db.inTransaction)
|
|
1678
|
+
this.db.pragma(`temp_store = ${prevTempStore}`);
|
|
1679
|
+
}
|
|
1680
|
+
finally {
|
|
1681
|
+
this.db.pragma("synchronous = NORMAL");
|
|
1682
|
+
}
|
|
1683
|
+
}
|
|
1684
|
+
}
|
|
1685
|
+
}
|
|
1686
|
+
/** Snapshot of the current bulk window's counters (inactive zeros when closed). */
|
|
1687
|
+
bulkStats() {
|
|
1688
|
+
if (this.bulk)
|
|
1689
|
+
return { ...this.bulk.stats };
|
|
1690
|
+
return {
|
|
1691
|
+
active: false,
|
|
1692
|
+
fresh: false,
|
|
1693
|
+
members: 0,
|
|
1694
|
+
reconcileSkipped: 0,
|
|
1695
|
+
reconcileProbed: 0,
|
|
1696
|
+
ftsDeleteSkipped: 0,
|
|
1697
|
+
};
|
|
1698
|
+
}
|
|
1699
|
+
// ─── Live-graph write batches (non-rebuild ingests) ────────────────────────
|
|
1700
|
+
/**
|
|
1701
|
+
* Group a non-rebuild ingest's per-member writes into one transaction per
|
|
1702
|
+
* BATCH_TXN_MEMBERS members or BATCH_TXN_MAX_MS, whichever comes first.
|
|
1703
|
+
* Unlike beginBulk nothing about durability changes: indexes stay, and
|
|
1704
|
+
* synchronous / journal_mode keep their values, so a crash loses at most the
|
|
1705
|
+
* open (uncommitted) batch and leaves the last committed state intact.
|
|
1706
|
+
*
|
|
1707
|
+
* Each transaction starts lazily at batchMemberStart with BEGIN IMMEDIATE (the
|
|
1708
|
+
* write lock is taken up front, waiting up to BUSY_TIMEOUT_MS; a deferred
|
|
1709
|
+
* BEGIN could fail every later member with SQLITE_BUSY_SNAPSHOT after another
|
|
1710
|
+
* connection's commit) and is committed only between members, so a
|
|
1711
|
+
* concurrent reader sees each member whole or not at all. An idle timer
|
|
1712
|
+
* commits a transaction left open while the member stream is slow (a live
|
|
1713
|
+
* org's retrieve), so the lock is never held much past maxMs. The writes are
|
|
1714
|
+
* the same statements as without a batch, so the graph is identical.
|
|
1715
|
+
*/
|
|
1716
|
+
beginBatch(opts = {}) {
|
|
1717
|
+
if (this.bulk)
|
|
1718
|
+
throw new StorageError("beginBatch: a bulk window is open");
|
|
1719
|
+
if (this.batch)
|
|
1720
|
+
throw new StorageError("beginBatch: a batch window is already open");
|
|
1721
|
+
if (this.db.inTransaction)
|
|
1722
|
+
throw new StorageError("beginBatch: called inside a transaction");
|
|
1723
|
+
this.batch = {
|
|
1724
|
+
maxMembers: Math.max(1, opts.maxMembers ?? BATCH_TXN_MEMBERS),
|
|
1725
|
+
maxMs: Math.max(1, opts.maxMs ?? BATCH_TXN_MAX_MS),
|
|
1726
|
+
open: false,
|
|
1727
|
+
openedAt: 0,
|
|
1728
|
+
inTxn: 0,
|
|
1729
|
+
timer: null,
|
|
1730
|
+
stats: { active: true, members: 0, commits: 0, beginFailures: 0 },
|
|
1731
|
+
};
|
|
1732
|
+
}
|
|
1733
|
+
/** Before a member's writes: open the batch transaction if none is open. */
|
|
1734
|
+
batchMemberStart() {
|
|
1735
|
+
const batch = this.batch;
|
|
1736
|
+
if (!batch || batch.open || this.db.inTransaction)
|
|
1737
|
+
return;
|
|
1738
|
+
try {
|
|
1739
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
1740
|
+
}
|
|
1741
|
+
catch {
|
|
1742
|
+
// Another writer kept the lock past busy_timeout: this member is
|
|
1743
|
+
// written per statement, exactly as without a batch.
|
|
1744
|
+
batch.stats.beginFailures += 1;
|
|
1745
|
+
return;
|
|
1746
|
+
}
|
|
1747
|
+
batch.open = true;
|
|
1748
|
+
batch.openedAt = Date.now();
|
|
1749
|
+
batch.inTxn = 0;
|
|
1750
|
+
const timer = setTimeout(() => {
|
|
1751
|
+
batch.timer = null;
|
|
1752
|
+
// Fires only while the event loop is idle, i.e. between members.
|
|
1753
|
+
if (this.batch === batch && batch.open && this.db.inTransaction) {
|
|
1754
|
+
try {
|
|
1755
|
+
this.commitBatch(batch);
|
|
1756
|
+
}
|
|
1757
|
+
catch {
|
|
1758
|
+
/* retried at the next member boundary */
|
|
1759
|
+
}
|
|
1760
|
+
}
|
|
1761
|
+
}, batch.maxMs);
|
|
1762
|
+
timer.unref?.();
|
|
1763
|
+
batch.timer = timer;
|
|
1764
|
+
}
|
|
1765
|
+
/** After a member's writes: commit when the batch is full or old enough. */
|
|
1766
|
+
batchMemberDone() {
|
|
1767
|
+
const batch = this.batch;
|
|
1768
|
+
if (!batch)
|
|
1769
|
+
return;
|
|
1770
|
+
batch.stats.members += 1;
|
|
1771
|
+
if (!batch.open)
|
|
1772
|
+
return;
|
|
1773
|
+
if (!this.db.inTransaction) {
|
|
1774
|
+
// SQLite rolled back the batch (SQLITE_FULL / IOERR / NOMEM inside a
|
|
1775
|
+
// member's savepoint): every member since the last commit is lost.
|
|
1776
|
+
this.clearBatchTimer(batch);
|
|
1777
|
+
batch.open = false;
|
|
1778
|
+
throw new StorageError("batch transaction was rolled back by SQLite; members written since the last commit were lost");
|
|
1779
|
+
}
|
|
1780
|
+
batch.inTxn += 1;
|
|
1781
|
+
if (batch.inTxn >= batch.maxMembers || Date.now() - batch.openedAt >= batch.maxMs) {
|
|
1782
|
+
this.commitBatch(batch);
|
|
1783
|
+
}
|
|
1784
|
+
}
|
|
1785
|
+
clearBatchTimer(batch) {
|
|
1786
|
+
if (batch.timer) {
|
|
1787
|
+
clearTimeout(batch.timer);
|
|
1788
|
+
batch.timer = null;
|
|
1789
|
+
}
|
|
1790
|
+
}
|
|
1791
|
+
commitBatch(batch) {
|
|
1792
|
+
this.clearBatchTimer(batch);
|
|
1793
|
+
if (batch.open && this.db.inTransaction) {
|
|
1794
|
+
this.db.exec("COMMIT");
|
|
1795
|
+
batch.stats.commits += 1;
|
|
1796
|
+
}
|
|
1797
|
+
batch.open = false;
|
|
1798
|
+
}
|
|
1799
|
+
/** Close the batch window, committing the open transaction. No-op when closed. */
|
|
1800
|
+
endBatch() {
|
|
1801
|
+
const batch = this.batch;
|
|
1802
|
+
if (!batch)
|
|
1803
|
+
return;
|
|
1804
|
+
this.batch = null;
|
|
1805
|
+
this.commitBatch(batch);
|
|
1806
|
+
}
|
|
1807
|
+
/** Snapshot of the current batch window's counters (inactive zeros when closed). */
|
|
1808
|
+
batchStats() {
|
|
1809
|
+
if (this.batch)
|
|
1810
|
+
return { ...this.batch.stats };
|
|
1811
|
+
return { active: false, members: 0, commits: 0, beginFailures: 0 };
|
|
1812
|
+
}
|
|
1813
|
+
searchNodesFts(orgId, query, k, opts = {}) {
|
|
1814
|
+
if (k <= 0)
|
|
1815
|
+
return [];
|
|
1816
|
+
// Build an OR of the query's word tokens (recall-oriented). splitIdentifierWords
|
|
1817
|
+
// applies the SAME camelCase/snake_case splitting the index body uses, so
|
|
1818
|
+
// `AccountController` matches the indexed "account controller". Each token is
|
|
1819
|
+
// a quoted FTS5 string literal so punctuation/operators can't be injected
|
|
1820
|
+
// into the MATCH grammar.
|
|
1821
|
+
const tokens = Array.from(new Set(splitIdentifierWords(query).map((t) => t.toLowerCase()))).slice(0, 32);
|
|
1822
|
+
if (tokens.length === 0)
|
|
1823
|
+
return [];
|
|
1824
|
+
const match = tokens.map((t) => `"${t}"`).join(" OR ");
|
|
1825
|
+
try {
|
|
1826
|
+
const rows = this.db
|
|
1827
|
+
.prepare(`SELECT qualified_name AS qname, label
|
|
1828
|
+
FROM _sfgraph_node_fts
|
|
1829
|
+
WHERE org_id = ? AND body MATCH ?${opts.label !== undefined ? " AND lower(label) = lower(?)" : ""}
|
|
1830
|
+
ORDER BY bm25(_sfgraph_node_fts)
|
|
1831
|
+
LIMIT ?`)
|
|
1832
|
+
.all(...(opts.label !== undefined ? [orgId, match, opts.label, k] : [orgId, match, k]));
|
|
1833
|
+
return rows;
|
|
1834
|
+
}
|
|
1835
|
+
catch {
|
|
1836
|
+
return [];
|
|
1837
|
+
}
|
|
1838
|
+
}
|
|
1839
|
+
// ─── Task 11c: delta ingestion ────────────────────────────────────────────
|
|
1840
|
+
//
|
|
1841
|
+
// `_sfgraph_members` records, per member of a filesystem source, its key
|
|
1842
|
+
// (sourceUri), fingerprint (file sizes + mtimes), content hash and the
|
|
1843
|
+
// qnames it owns (nodes it emitted plus the sources of edges it emitted).
|
|
1844
|
+
// `_sfgraph_delta_pending` holds the qnames an in-flight delta is
|
|
1845
|
+
// rewriting; it is cleared when the delta completes, so a leftover row
|
|
1846
|
+
// means the previous delta was interrupted. `_sfgraph_member_generation`
|
|
1847
|
+
// records which sfgraph build wrote the rows: a delta under a different
|
|
1848
|
+
// build would mix old and new parser output, so it rebuilds instead. All
|
|
1849
|
+
// are created on first use (no schema-version bump: an older sfgraph simply
|
|
1850
|
+
// ignores them).
|
|
1851
|
+
deltaTablesReady = false;
|
|
1852
|
+
ensureDeltaTables() {
|
|
1853
|
+
if (this.deltaTablesReady)
|
|
1854
|
+
return;
|
|
1855
|
+
this.db.exec(`CREATE TABLE IF NOT EXISTS _sfgraph_members (
|
|
1856
|
+
org_id TEXT NOT NULL,
|
|
1857
|
+
member_key TEXT NOT NULL,
|
|
1858
|
+
fingerprint TEXT NOT NULL,
|
|
1859
|
+
hash TEXT NOT NULL,
|
|
1860
|
+
owned TEXT NOT NULL,
|
|
1861
|
+
PRIMARY KEY (org_id, member_key)
|
|
1862
|
+
);
|
|
1863
|
+
CREATE TABLE IF NOT EXISTS _sfgraph_member_generation (
|
|
1864
|
+
org_id TEXT PRIMARY KEY,
|
|
1865
|
+
generation TEXT NOT NULL
|
|
1866
|
+
);
|
|
1867
|
+
CREATE TABLE IF NOT EXISTS _sfgraph_delta_pending (
|
|
1868
|
+
org_id TEXT PRIMARY KEY,
|
|
1869
|
+
qnames TEXT NOT NULL,
|
|
1870
|
+
started_at INTEGER NOT NULL
|
|
1871
|
+
);`);
|
|
1872
|
+
this.deltaTablesReady = true;
|
|
1873
|
+
}
|
|
1874
|
+
listMemberRows(orgId) {
|
|
1875
|
+
this.ensureDeltaTables();
|
|
1876
|
+
const rows = this.stmt("SELECT member_key, fingerprint, hash, owned FROM _sfgraph_members WHERE org_id = ?").all(orgId);
|
|
1877
|
+
return rows.map((r) => {
|
|
1878
|
+
let owned = [];
|
|
1879
|
+
try {
|
|
1880
|
+
const v = JSON.parse(r.owned);
|
|
1881
|
+
if (Array.isArray(v))
|
|
1882
|
+
owned = v.filter((x) => typeof x === "string");
|
|
1883
|
+
}
|
|
1884
|
+
catch {
|
|
1885
|
+
owned = [];
|
|
1886
|
+
}
|
|
1887
|
+
return { key: r.member_key, fingerprint: r.fingerprint, hash: r.hash, owned };
|
|
1888
|
+
});
|
|
1889
|
+
}
|
|
1890
|
+
/**
|
|
1891
|
+
* Replace every member row of the org (a full ingest), stamped with the
|
|
1892
|
+
* `generation` (sfgraph build) whose parsers produced them.
|
|
1893
|
+
*/
|
|
1894
|
+
replaceMemberRows(orgId, rows, generation) {
|
|
1895
|
+
this.ensureDeltaTables();
|
|
1896
|
+
this.db.transaction(() => {
|
|
1897
|
+
this.stmt("DELETE FROM _sfgraph_members WHERE org_id = ?").run(orgId);
|
|
1898
|
+
this.writeMemberRows(orgId, rows);
|
|
1899
|
+
if (generation === undefined) {
|
|
1900
|
+
this.stmt("DELETE FROM _sfgraph_member_generation WHERE org_id = ?").run(orgId);
|
|
1901
|
+
}
|
|
1902
|
+
else {
|
|
1903
|
+
this.stmt(`INSERT INTO _sfgraph_member_generation(org_id, generation) VALUES (?, ?)
|
|
1904
|
+
ON CONFLICT(org_id) DO UPDATE SET generation = excluded.generation`).run(orgId, generation);
|
|
1905
|
+
}
|
|
1906
|
+
})();
|
|
1907
|
+
}
|
|
1908
|
+
/** The generation the org's member rows were written by, or null when unknown. */
|
|
1909
|
+
getMemberRowsGeneration(orgId) {
|
|
1910
|
+
this.ensureDeltaTables();
|
|
1911
|
+
const r = this.stmt("SELECT generation FROM _sfgraph_member_generation WHERE org_id = ?").get(orgId);
|
|
1912
|
+
return r?.generation ?? null;
|
|
1913
|
+
}
|
|
1914
|
+
upsertMemberRows(orgId, rows) {
|
|
1915
|
+
this.ensureDeltaTables();
|
|
1916
|
+
this.db.transaction(() => this.writeMemberRows(orgId, rows))();
|
|
1917
|
+
}
|
|
1918
|
+
writeMemberRows(orgId, rows) {
|
|
1919
|
+
const st = this.stmt(`INSERT INTO _sfgraph_members(org_id, member_key, fingerprint, hash, owned) VALUES (?, ?, ?, ?, ?)
|
|
1920
|
+
ON CONFLICT(org_id, member_key) DO UPDATE SET fingerprint = excluded.fingerprint,
|
|
1921
|
+
hash = excluded.hash, owned = excluded.owned`);
|
|
1922
|
+
for (const r of rows)
|
|
1923
|
+
st.run(orgId, r.key, r.fingerprint, r.hash, JSON.stringify(r.owned));
|
|
1924
|
+
}
|
|
1925
|
+
deleteMemberRows(orgId, keys) {
|
|
1926
|
+
this.ensureDeltaTables();
|
|
1927
|
+
const st = this.stmt("DELETE FROM _sfgraph_members WHERE org_id = ? AND member_key = ?");
|
|
1928
|
+
this.db.transaction(() => {
|
|
1929
|
+
for (const k of keys)
|
|
1930
|
+
st.run(orgId, k);
|
|
1931
|
+
})();
|
|
1932
|
+
}
|
|
1933
|
+
clearMemberRows(orgId) {
|
|
1934
|
+
this.ensureDeltaTables();
|
|
1935
|
+
// Rows and their generation stamp go together (final review core M-4).
|
|
1936
|
+
this.db.transaction(() => {
|
|
1937
|
+
this.stmt("DELETE FROM _sfgraph_members WHERE org_id = ?").run(orgId);
|
|
1938
|
+
this.stmt("DELETE FROM _sfgraph_member_generation WHERE org_id = ?").run(orgId);
|
|
1939
|
+
})();
|
|
1940
|
+
}
|
|
1941
|
+
getDeltaPending(orgId) {
|
|
1942
|
+
this.ensureDeltaTables();
|
|
1943
|
+
const row = this.stmt("SELECT qnames FROM _sfgraph_delta_pending WHERE org_id = ?").get(orgId);
|
|
1944
|
+
if (!row)
|
|
1945
|
+
return null;
|
|
1946
|
+
try {
|
|
1947
|
+
const v = JSON.parse(row.qnames);
|
|
1948
|
+
return Array.isArray(v) ? v.filter((x) => typeof x === "string") : [];
|
|
1949
|
+
}
|
|
1950
|
+
catch {
|
|
1951
|
+
return [];
|
|
1952
|
+
}
|
|
1953
|
+
}
|
|
1954
|
+
setDeltaPending(orgId, qnames) {
|
|
1955
|
+
this.ensureDeltaTables();
|
|
1956
|
+
this.stmt(`INSERT INTO _sfgraph_delta_pending(org_id, qnames, started_at) VALUES (?, ?, ?)
|
|
1957
|
+
ON CONFLICT(org_id) DO UPDATE SET qnames = excluded.qnames, started_at = excluded.started_at`).run(orgId, JSON.stringify(qnames), Date.now());
|
|
1958
|
+
}
|
|
1959
|
+
clearDeltaPending(orgId) {
|
|
1960
|
+
this.ensureDeltaTables();
|
|
1961
|
+
this.stmt("DELETE FROM _sfgraph_delta_pending WHERE org_id = ?").run(orgId);
|
|
1962
|
+
}
|
|
1963
|
+
/** The subset of `qnames` that exist as nodes (stored spelling). */
|
|
1964
|
+
existingQnames(orgId, qnames) {
|
|
1965
|
+
const out = new Set();
|
|
1966
|
+
for (const slice of chunks([...qnames], 400)) {
|
|
1967
|
+
const rows = this.stmt(`SELECT qualified_name FROM _sfgraph_node_index WHERE org_id = ? AND qualified_name IN (${slice.map(() => "?").join(",")})`).all(orgId, ...slice);
|
|
1968
|
+
for (const r of rows)
|
|
1969
|
+
out.add(r.qualified_name);
|
|
1970
|
+
}
|
|
1971
|
+
return out;
|
|
1972
|
+
}
|
|
1973
|
+
/** Stored spellings of the nodes matching `qnames` case-insensitively. */
|
|
1974
|
+
existingQnamesNocase(orgId, qnames) {
|
|
1975
|
+
const out = new Set();
|
|
1976
|
+
for (const slice of chunks([...qnames], 400)) {
|
|
1977
|
+
const rows = this.stmt(`SELECT qualified_name FROM _sfgraph_node_index WHERE org_id = ? AND qualified_name COLLATE NOCASE IN (${slice.map(() => "?").join(",")})`).all(orgId, ...slice);
|
|
1978
|
+
for (const r of rows)
|
|
1979
|
+
out.add(r.qualified_name);
|
|
1980
|
+
}
|
|
1981
|
+
return out;
|
|
1982
|
+
}
|
|
1983
|
+
/** Overwrite the source_hash of existing nodes (delta's stale sentinel). */
|
|
1984
|
+
setNodeSourceHash(orgId, qnames, hash) {
|
|
1985
|
+
this.syncCaches();
|
|
1986
|
+
let n = 0;
|
|
1987
|
+
this.db.transaction(() => {
|
|
1988
|
+
for (const slice of chunks([...qnames], 400)) {
|
|
1989
|
+
const rows = this.stmt(`SELECT qualified_name, label FROM _sfgraph_node_index WHERE org_id = ? AND qualified_name IN (${slice.map(() => "?").join(",")})`).all(orgId, ...slice);
|
|
1990
|
+
for (const r of rows) {
|
|
1991
|
+
const tbl = this.nodeLabelCache.get(r.label);
|
|
1992
|
+
if (!tbl)
|
|
1993
|
+
continue;
|
|
1994
|
+
n += this.stmt(`UPDATE ${tbl} SET source_hash = ? WHERE org_id = ? AND qualified_name = ?`).run(hash, orgId, r.qualified_name).changes;
|
|
1995
|
+
}
|
|
1996
|
+
}
|
|
1997
|
+
})();
|
|
1998
|
+
return n;
|
|
1999
|
+
}
|
|
2000
|
+
/** source_hash and last_modified_at of existing nodes, keyed by stored qname. */
|
|
2001
|
+
getNodeStamps(orgId, qnames) {
|
|
2002
|
+
this.syncCaches();
|
|
2003
|
+
const out = new Map();
|
|
2004
|
+
for (const slice of chunks([...qnames], 400)) {
|
|
2005
|
+
const rows = this.stmt(`SELECT qualified_name, label FROM _sfgraph_node_index WHERE org_id = ? AND qualified_name IN (${slice.map(() => "?").join(",")})`).all(orgId, ...slice);
|
|
2006
|
+
for (const r of rows) {
|
|
2007
|
+
const tbl = this.nodeLabelCache.get(r.label);
|
|
2008
|
+
if (!tbl)
|
|
2009
|
+
continue;
|
|
2010
|
+
const row = this.stmt(`SELECT source_hash AS h, last_modified_at AS m FROM ${tbl} WHERE org_id = ? AND qualified_name = ?`).get(orgId, r.qualified_name);
|
|
2011
|
+
if (row)
|
|
2012
|
+
out.set(r.qualified_name, { hash: row.h, lastModifiedAt: row.m });
|
|
2013
|
+
}
|
|
2014
|
+
}
|
|
2015
|
+
return out;
|
|
2016
|
+
}
|
|
2017
|
+
/**
|
|
2018
|
+
* Put recorded stamps back. `when: "hash-is"` restores both fields on nodes
|
|
2019
|
+
* whose source_hash is currently `hash` (a sentinel nobody re-emitted);
|
|
2020
|
+
* `when: "unchanged"` restores last_modified_at on nodes whose source_hash
|
|
2021
|
+
* equals the recorded one (re-merged with identical content).
|
|
2022
|
+
*/
|
|
2023
|
+
restoreNodeStamps(orgId, stamps, cond) {
|
|
2024
|
+
this.syncCaches();
|
|
2025
|
+
let n = 0;
|
|
2026
|
+
this.db.transaction(() => {
|
|
2027
|
+
for (const slice of chunks([...stamps.keys()], 400)) {
|
|
2028
|
+
const rows = this.stmt(`SELECT qualified_name, label FROM _sfgraph_node_index WHERE org_id = ? AND qualified_name IN (${slice.map(() => "?").join(",")})`).all(orgId, ...slice);
|
|
2029
|
+
for (const r of rows) {
|
|
2030
|
+
const tbl = this.nodeLabelCache.get(r.label);
|
|
2031
|
+
const st = stamps.get(r.qualified_name);
|
|
2032
|
+
if (!tbl || !st)
|
|
2033
|
+
continue;
|
|
2034
|
+
n +=
|
|
2035
|
+
cond.when === "hash-is"
|
|
2036
|
+
? this.stmt(`UPDATE ${tbl} SET source_hash = ?, last_modified_at = ? WHERE org_id = ? AND qualified_name = ? AND source_hash = ?`).run(st.hash, st.lastModifiedAt, orgId, r.qualified_name, cond.hash).changes
|
|
2037
|
+
: this.stmt(`UPDATE ${tbl} SET last_modified_at = ? WHERE org_id = ? AND qualified_name = ? AND source_hash = ? AND last_modified_at <> ?`).run(st.lastModifiedAt, orgId, r.qualified_name, st.hash, st.lastModifiedAt)
|
|
2038
|
+
.changes;
|
|
2039
|
+
}
|
|
2040
|
+
}
|
|
2041
|
+
})();
|
|
2042
|
+
return n;
|
|
2043
|
+
}
|
|
2044
|
+
listNodesWithSourceHash(orgId, hash) {
|
|
2045
|
+
this.syncCaches();
|
|
2046
|
+
const out = [];
|
|
2047
|
+
for (const tbl of this.nodeLabelCache.values()) {
|
|
2048
|
+
const rows = this.stmt(`SELECT qualified_name FROM ${tbl} WHERE org_id = ? AND source_hash = ?`).all(orgId, hash);
|
|
2049
|
+
for (const r of rows)
|
|
2050
|
+
out.push(r.qualified_name);
|
|
2051
|
+
}
|
|
2052
|
+
return out;
|
|
2053
|
+
}
|
|
2054
|
+
/** Delete every outgoing edge of `srcs` (any pass); returns what was deleted. */
|
|
2055
|
+
deleteEdgesFrom(orgId, srcs) {
|
|
2056
|
+
this.syncCaches();
|
|
2057
|
+
const out = [];
|
|
2058
|
+
const all = [...srcs];
|
|
2059
|
+
if (all.length === 0)
|
|
2060
|
+
return out;
|
|
2061
|
+
this.db.transaction(() => {
|
|
2062
|
+
for (const [relType, tbl] of this.edgeRelCache) {
|
|
2063
|
+
for (const slice of chunks(all, 400)) {
|
|
2064
|
+
const ph = slice.map(() => "?").join(",");
|
|
2065
|
+
const rows = this.stmt(`SELECT src_qname, dst_qname, first_seen_at FROM ${tbl} WHERE org_id = ? AND src_qname IN (${ph})`).all(orgId, ...slice);
|
|
2066
|
+
if (rows.length === 0)
|
|
2067
|
+
continue;
|
|
2068
|
+
this.stmt(`DELETE FROM ${tbl} WHERE org_id = ? AND src_qname IN (${ph})`).run(orgId, ...slice);
|
|
2069
|
+
for (const r of rows) {
|
|
2070
|
+
out.push({ relType, src: r.src_qname, dst: r.dst_qname, firstSeenAt: r.first_seen_at });
|
|
2071
|
+
}
|
|
2072
|
+
}
|
|
2073
|
+
}
|
|
2074
|
+
})();
|
|
2075
|
+
return out;
|
|
2076
|
+
}
|
|
2077
|
+
/** Edges (any rel type) whose dst is one of `dsts`. */
|
|
2078
|
+
listEdgesToAny(orgId, dsts) {
|
|
2079
|
+
this.syncCaches();
|
|
2080
|
+
const out = [];
|
|
2081
|
+
const all = [...dsts];
|
|
2082
|
+
if (all.length === 0)
|
|
2083
|
+
return out;
|
|
2084
|
+
for (const [relType, tbl] of this.edgeRelCache) {
|
|
2085
|
+
for (const slice of chunks(all, 400)) {
|
|
2086
|
+
const rows = this.stmt(`SELECT org_id, src_qname, dst_qname, attributes, first_seen_at, last_seen_at FROM ${tbl} WHERE org_id = ? AND dst_qname IN (${slice.map(() => "?").join(",")})`).all(orgId, ...slice);
|
|
2087
|
+
for (const r of rows)
|
|
2088
|
+
out.push(rowToEdge(r, relType));
|
|
2089
|
+
}
|
|
2090
|
+
}
|
|
2091
|
+
return out;
|
|
2092
|
+
}
|
|
2093
|
+
/** Edges of `relType` whose src is one of `srcs` (NOCASE, like listEdgesFrom). */
|
|
2094
|
+
listEdgesFromAny(orgId, srcs, relType) {
|
|
2095
|
+
this.syncCaches();
|
|
2096
|
+
const out = [];
|
|
2097
|
+
const tbl = this.edgeRelCache.get(relType);
|
|
2098
|
+
const all = [...srcs];
|
|
2099
|
+
if (!tbl || all.length === 0)
|
|
2100
|
+
return out;
|
|
2101
|
+
for (const slice of chunks(all, 400)) {
|
|
2102
|
+
const rows = this.stmt(`SELECT org_id, src_qname, dst_qname, attributes, first_seen_at, last_seen_at FROM ${tbl} WHERE org_id = ? AND src_qname IN (${slice.map(() => "?").join(",")})`).all(orgId, ...slice);
|
|
2103
|
+
for (const r of rows)
|
|
2104
|
+
out.push(rowToEdge(r, relType));
|
|
2105
|
+
}
|
|
2106
|
+
return out;
|
|
2107
|
+
}
|
|
2108
|
+
/**
|
|
2109
|
+
* Whether any node of `labels` has attributes JSON text containing one of
|
|
2110
|
+
* `needles` (a plain substring test, so a superset of "has that key").
|
|
2111
|
+
* Reads the text only, no JSON parse; stops at the first hit.
|
|
2112
|
+
*/
|
|
2113
|
+
anyNodeAttributesContain(orgId, labels, needles) {
|
|
2114
|
+
this.syncCaches();
|
|
2115
|
+
if (needles.length === 0)
|
|
2116
|
+
return false;
|
|
2117
|
+
const cond = needles.map(() => "instr(attributes, ?) > 0").join(" OR ");
|
|
2118
|
+
for (const label of labels) {
|
|
2119
|
+
const tbl = this.nodeLabelCache.get(label);
|
|
2120
|
+
if (!tbl)
|
|
2121
|
+
continue;
|
|
2122
|
+
if (this.stmt(`SELECT 1 FROM ${tbl} WHERE org_id = ? AND (${cond}) LIMIT 1`).get(orgId, ...needles))
|
|
2123
|
+
return true;
|
|
2124
|
+
}
|
|
2125
|
+
return false;
|
|
2126
|
+
}
|
|
2127
|
+
/** `anyNodeAttributesContain` for the edges of `relTypes`. */
|
|
2128
|
+
anyEdgeAttributesContain(orgId, relTypes, needle) {
|
|
2129
|
+
this.syncCaches();
|
|
2130
|
+
for (const rel of relTypes) {
|
|
2131
|
+
const tbl = this.edgeRelCache.get(rel);
|
|
2132
|
+
if (!tbl)
|
|
2133
|
+
continue;
|
|
2134
|
+
if (this.stmt(`SELECT 1 FROM ${tbl} WHERE org_id = ? AND instr(attributes, ?) > 0 LIMIT 1`).get(orgId, needle))
|
|
2135
|
+
return true;
|
|
2136
|
+
}
|
|
2137
|
+
return false;
|
|
2138
|
+
}
|
|
2139
|
+
/**
|
|
2140
|
+
* Sum of `attributes[attr]` over the nodes of `labels` where it is a
|
|
2141
|
+
* number > 0, in listNodesByLabel order (label by label).
|
|
2142
|
+
*/
|
|
2143
|
+
sumPositiveNodeAttribute(orgId, labels, attr) {
|
|
2144
|
+
this.syncCaches();
|
|
2145
|
+
const path = `$.${JSON.stringify(attr)}`;
|
|
2146
|
+
let sum = 0;
|
|
2147
|
+
for (const label of labels) {
|
|
2148
|
+
const tbl = this.nodeLabelCache.get(label);
|
|
2149
|
+
if (!tbl)
|
|
2150
|
+
continue;
|
|
2151
|
+
const rows = this.stmt(`SELECT json_type(attributes, ?) AS t, json_extract(attributes, ?) AS v FROM ${tbl} WHERE org_id = ? AND instr(attributes, ?) > 0`).all(path, path, orgId, JSON.stringify(attr));
|
|
2152
|
+
for (const r of rows) {
|
|
2153
|
+
if ((r.t === "integer" || r.t === "real") && typeof r.v === "number" && r.v > 0)
|
|
2154
|
+
sum += r.v;
|
|
2155
|
+
}
|
|
2156
|
+
}
|
|
2157
|
+
return sum;
|
|
2158
|
+
}
|
|
2159
|
+
/**
|
|
2160
|
+
* Stored qnames (as `_sfgraph_node_index` spells them) of the org's nodes
|
|
2161
|
+
* whose `attributes[attr]` is JSON `true` — exactly the nodes for which
|
|
2162
|
+
* `getNode(q)?.attributes[attr] === true`, read in one pass per label.
|
|
2163
|
+
*/
|
|
2164
|
+
listNodesWithTrueAttribute(orgId, attr) {
|
|
2165
|
+
this.syncCaches();
|
|
2166
|
+
const out = new Set();
|
|
2167
|
+
const path = `$.${JSON.stringify(attr)}`;
|
|
2168
|
+
for (const [label, tbl] of this.nodeLabelCache) {
|
|
2169
|
+
const rows = this.stmt(`SELECT i.qualified_name AS q FROM _sfgraph_node_index i
|
|
2170
|
+
JOIN ${tbl} t ON t.org_id = i.org_id AND t.qualified_name = i.qualified_name
|
|
2171
|
+
WHERE i.org_id = ? AND i.label = ? AND json_type(t.attributes, ?) = 'true'`).all(orgId, label, path);
|
|
2172
|
+
for (const r of rows)
|
|
2173
|
+
out.add(r.q);
|
|
2174
|
+
}
|
|
2175
|
+
return out;
|
|
2176
|
+
}
|
|
2177
|
+
/**
|
|
2178
|
+
* Delete reflection-walker edges (`source: "reflection"` REFERENCES) of the
|
|
2179
|
+
* org: those FROM `srcs` and/or INTO `dsts`; with neither, all of them.
|
|
2180
|
+
*/
|
|
2181
|
+
deleteReflectionEdges(orgId, scope = {}) {
|
|
2182
|
+
this.syncCaches();
|
|
2183
|
+
const tbl = this.edgeRelCache.get("REFERENCES");
|
|
2184
|
+
if (!tbl)
|
|
2185
|
+
return 0;
|
|
2186
|
+
const isRefl = "json_extract(attributes, '$.source') = 'reflection'";
|
|
2187
|
+
let n = 0;
|
|
2188
|
+
this.db.transaction(() => {
|
|
2189
|
+
if (!scope.srcs && !scope.dsts) {
|
|
2190
|
+
n += this.stmt(`DELETE FROM ${tbl} WHERE org_id = ? AND ${isRefl}`).run(orgId).changes;
|
|
2191
|
+
return;
|
|
2192
|
+
}
|
|
2193
|
+
for (const [col, list] of [
|
|
2194
|
+
["src_qname", scope.srcs],
|
|
2195
|
+
["dst_qname", scope.dsts],
|
|
2196
|
+
]) {
|
|
2197
|
+
if (!list)
|
|
2198
|
+
continue;
|
|
2199
|
+
for (const slice of chunks([...list], 400)) {
|
|
2200
|
+
n += this.stmt(`DELETE FROM ${tbl} WHERE org_id = ? AND ${col} IN (${slice.map(() => "?").join(",")}) AND ${isRefl}`).run(orgId, ...slice).changes;
|
|
2201
|
+
}
|
|
2202
|
+
}
|
|
2203
|
+
})();
|
|
2204
|
+
return n;
|
|
2205
|
+
}
|
|
2206
|
+
/**
|
|
2207
|
+
* Qnames of nodes whose attributes JSON contains one of `needles` as a whole
|
|
2208
|
+
* JSON string (case-insensitive for ASCII). A superset of the nodes whose
|
|
2209
|
+
* attribute string values equal a needle — the reflection walker's input.
|
|
2210
|
+
*/
|
|
2211
|
+
findNodesMentioning(orgId, needles) {
|
|
2212
|
+
this.syncCaches();
|
|
2213
|
+
const out = new Set();
|
|
2214
|
+
if (needles.length === 0)
|
|
2215
|
+
return out;
|
|
2216
|
+
const likeEsc = (s) => s.replace(/[\\%_]/g, (c) => `\\${c}`);
|
|
2217
|
+
const pats = needles.map((nd) => `%${likeEsc(JSON.stringify(nd))}%`);
|
|
2218
|
+
for (const tbl of this.nodeLabelCache.values()) {
|
|
2219
|
+
for (const slice of chunks(pats, 50)) {
|
|
2220
|
+
const rows = this.stmt(`SELECT qualified_name FROM ${tbl} WHERE org_id = ? AND (${slice.map(() => "attributes LIKE ? ESCAPE '\\'").join(" OR ")})`).all(orgId, ...slice);
|
|
2221
|
+
for (const r of rows)
|
|
2222
|
+
out.add(r.qualified_name);
|
|
2223
|
+
}
|
|
2224
|
+
}
|
|
2225
|
+
return out;
|
|
2226
|
+
}
|
|
2227
|
+
/** `src|dst` (lower-cased) of every resolver-emitted (`resolvedBy`) REFERENCES edge. */
|
|
2228
|
+
listResolverReferencePairs(orgId) {
|
|
2229
|
+
this.syncCaches();
|
|
2230
|
+
const out = new Set();
|
|
2231
|
+
const tbl = this.edgeRelCache.get("REFERENCES");
|
|
2232
|
+
if (!tbl)
|
|
2233
|
+
return out;
|
|
2234
|
+
const rows = this.stmt(`SELECT src_qname, dst_qname FROM ${tbl} WHERE org_id = ? AND json_extract(attributes, '$.resolvedBy') IS NOT NULL`).all(orgId);
|
|
2235
|
+
for (const r of rows)
|
|
2236
|
+
out.add(`${r.src_qname}${KSEP}${r.dst_qname}`.toLowerCase());
|
|
2237
|
+
return out;
|
|
2238
|
+
}
|
|
2239
|
+
/** `src|dst` (lower-cased) of every CANONICAL_OF edge. */
|
|
2240
|
+
listCanonicalPairs(orgId) {
|
|
2241
|
+
this.syncCaches();
|
|
2242
|
+
const out = new Set();
|
|
2243
|
+
const tbl = this.edgeRelCache.get("CANONICAL_OF");
|
|
2244
|
+
if (!tbl)
|
|
2245
|
+
return out;
|
|
2246
|
+
const rows = this.stmt(`SELECT src_qname, dst_qname FROM ${tbl} WHERE org_id = ?`).all(orgId);
|
|
2247
|
+
for (const r of rows)
|
|
2248
|
+
out.add(`${r.src_qname}${KSEP}${r.dst_qname}`.toLowerCase());
|
|
2249
|
+
return out;
|
|
2250
|
+
}
|
|
2251
|
+
/** Lower an edge's first_seen_at to `firstSeenAt` (re-created edges keep their history). */
|
|
2252
|
+
restoreEdgeFirstSeen(orgId, edges) {
|
|
2253
|
+
this.syncCaches();
|
|
2254
|
+
this.db.transaction(() => {
|
|
2255
|
+
for (const e of edges) {
|
|
2256
|
+
const tbl = this.edgeRelCache.get(e.relType);
|
|
2257
|
+
if (!tbl)
|
|
2258
|
+
continue;
|
|
2259
|
+
this.stmt(`UPDATE ${tbl} SET first_seen_at = ? WHERE org_id = ? AND src_qname = ? AND dst_qname = ? AND first_seen_at > ?`).run(e.firstSeenAt, orgId, e.src, e.dst, e.firstSeenAt);
|
|
2260
|
+
}
|
|
2261
|
+
})();
|
|
2262
|
+
}
|
|
579
2263
|
// Helpers for shared usage
|
|
580
2264
|
getNodeLabelTables() {
|
|
2265
|
+
this.syncCaches();
|
|
581
2266
|
return new Map(this.nodeLabelCache);
|
|
582
2267
|
}
|
|
583
2268
|
getEdgeRelTables() {
|
|
2269
|
+
this.syncCaches();
|
|
584
2270
|
return new Map(this.edgeRelCache);
|
|
585
2271
|
}
|
|
586
2272
|
// For tests/diagnostics
|
|
587
2273
|
_explainReverseEdgeQuery(relType) {
|
|
2274
|
+
this.syncCaches();
|
|
588
2275
|
const tbl = this.edgeRelCache.get(relType);
|
|
589
2276
|
if (!tbl)
|
|
590
2277
|
throw new StorageError(`unknown rel type ${relType}`);
|