@ryanstark24/sfgraph-core 1.2.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/analyze/apex-platform.d.ts +23 -0
- package/dist/analyze/apex-platform.d.ts.map +1 -0
- package/dist/analyze/apex-platform.js +112 -0
- package/dist/analyze/apex-platform.js.map +1 -0
- package/dist/analyze/audit-graph.d.ts +51 -3
- package/dist/analyze/audit-graph.d.ts.map +1 -1
- package/dist/analyze/audit-graph.js +382 -14
- package/dist/analyze/audit-graph.js.map +1 -1
- package/dist/analyze/component-scope.d.ts +82 -0
- package/dist/analyze/component-scope.d.ts.map +1 -0
- package/dist/analyze/component-scope.js +207 -0
- package/dist/analyze/component-scope.js.map +1 -0
- package/dist/analyze/containment.d.ts +91 -0
- package/dist/analyze/containment.d.ts.map +1 -0
- package/dist/analyze/containment.js +175 -0
- package/dist/analyze/containment.js.map +1 -0
- package/dist/analyze/cross-org.d.ts +35 -0
- package/dist/analyze/cross-org.d.ts.map +1 -1
- package/dist/analyze/cross-org.js +102 -3
- package/dist/analyze/cross-org.js.map +1 -1
- package/dist/analyze/dead-code.d.ts +99 -3
- package/dist/analyze/dead-code.d.ts.map +1 -1
- package/dist/analyze/dead-code.js +282 -10
- package/dist/analyze/dead-code.js.map +1 -1
- package/dist/analyze/dependencies.d.ts +21 -2
- package/dist/analyze/dependencies.d.ts.map +1 -1
- package/dist/analyze/dependencies.js +102 -12
- package/dist/analyze/dependencies.js.map +1 -1
- package/dist/analyze/dependents.d.ts +116 -1
- package/dist/analyze/dependents.d.ts.map +1 -1
- package/dist/analyze/dependents.js +285 -13
- package/dist/analyze/dependents.js.map +1 -1
- package/dist/analyze/deployment-manifest/generate.d.ts +24 -0
- package/dist/analyze/deployment-manifest/generate.d.ts.map +1 -1
- package/dist/analyze/deployment-manifest/generate.js +40 -12
- package/dist/analyze/deployment-manifest/generate.js.map +1 -1
- package/dist/analyze/deployment-manifest/member-name-formatters.d.ts.map +1 -1
- package/dist/analyze/deployment-manifest/member-name-formatters.js +1 -0
- package/dist/analyze/deployment-manifest/member-name-formatters.js.map +1 -1
- package/dist/analyze/deployment-manifest/member-types.d.ts.map +1 -1
- package/dist/analyze/deployment-manifest/member-types.js +2 -0
- package/dist/analyze/deployment-manifest/member-types.js.map +1 -1
- package/dist/analyze/diff-seeds.d.ts +33 -0
- package/dist/analyze/diff-seeds.d.ts.map +1 -0
- package/dist/analyze/diff-seeds.js +299 -0
- package/dist/analyze/diff-seeds.js.map +1 -0
- package/dist/analyze/edge-filters.d.ts +62 -0
- package/dist/analyze/edge-filters.d.ts.map +1 -0
- package/dist/analyze/edge-filters.js +127 -0
- package/dist/analyze/edge-filters.js.map +1 -0
- package/dist/analyze/field-scope.d.ts +25 -0
- package/dist/analyze/field-scope.d.ts.map +1 -0
- package/dist/analyze/field-scope.js +94 -0
- package/dist/analyze/field-scope.js.map +1 -0
- package/dist/analyze/find-nodes.d.ts +48 -12
- package/dist/analyze/find-nodes.d.ts.map +1 -1
- package/dist/analyze/find-nodes.js +323 -12
- package/dist/analyze/find-nodes.js.map +1 -1
- package/dist/analyze/findings.d.ts +1 -1
- package/dist/analyze/findings.d.ts.map +1 -1
- package/dist/analyze/findings.js +15 -3
- package/dist/analyze/findings.js.map +1 -1
- package/dist/analyze/git-diff.d.ts +27 -1
- package/dist/analyze/git-diff.d.ts.map +1 -1
- package/dist/analyze/git-diff.js +198 -26
- package/dist/analyze/git-diff.js.map +1 -1
- package/dist/analyze/governor.d.ts +25 -4
- package/dist/analyze/governor.d.ts.map +1 -1
- package/dist/analyze/governor.js +82 -18
- package/dist/analyze/governor.js.map +1 -1
- package/dist/analyze/index.d.ts +6 -0
- package/dist/analyze/index.d.ts.map +1 -1
- package/dist/analyze/index.js +6 -0
- package/dist/analyze/index.js.map +1 -1
- package/dist/analyze/path-to-qname.d.ts +9 -0
- package/dist/analyze/path-to-qname.d.ts.map +1 -1
- package/dist/analyze/path-to-qname.js +58 -15
- package/dist/analyze/path-to-qname.js.map +1 -1
- package/dist/analyze/populate.d.ts +40 -0
- package/dist/analyze/populate.d.ts.map +1 -1
- package/dist/analyze/populate.js +203 -58
- package/dist/analyze/populate.js.map +1 -1
- package/dist/analyze/pseudo-endpoints.d.ts +67 -0
- package/dist/analyze/pseudo-endpoints.d.ts.map +1 -0
- package/dist/analyze/pseudo-endpoints.js +78 -0
- package/dist/analyze/pseudo-endpoints.js.map +1 -0
- package/dist/analyze/security.d.ts +17 -0
- package/dist/analyze/security.d.ts.map +1 -1
- package/dist/analyze/security.js +103 -18
- package/dist/analyze/security.js.map +1 -1
- package/dist/analyze/test-coverage.d.ts +48 -0
- package/dist/analyze/test-coverage.d.ts.map +1 -1
- package/dist/analyze/test-coverage.js +72 -1
- package/dist/analyze/test-coverage.js.map +1 -1
- package/dist/analyze/wip-changed-files.d.ts +88 -0
- package/dist/analyze/wip-changed-files.d.ts.map +1 -0
- package/dist/analyze/wip-changed-files.js +422 -0
- package/dist/analyze/wip-changed-files.js.map +1 -0
- package/dist/analyze/wip-impact.d.ts +59 -0
- package/dist/analyze/wip-impact.d.ts.map +1 -1
- package/dist/analyze/wip-impact.js +199 -116
- package/dist/analyze/wip-impact.js.map +1 -1
- package/dist/domain/apex-platform.d.ts +10 -0
- package/dist/domain/apex-platform.d.ts.map +1 -0
- package/dist/domain/apex-platform.js +231 -0
- package/dist/domain/apex-platform.js.map +1 -0
- package/dist/domain/index.d.ts +2 -0
- package/dist/domain/index.d.ts.map +1 -1
- package/dist/domain/index.js +2 -0
- package/dist/domain/index.js.map +1 -1
- package/dist/domain/metadata-category.d.ts +1 -0
- package/dist/domain/metadata-category.d.ts.map +1 -1
- package/dist/domain/metadata-category.js +1 -0
- package/dist/domain/metadata-category.js.map +1 -1
- package/dist/domain/rel-types.d.ts +7 -0
- package/dist/domain/rel-types.d.ts.map +1 -1
- package/dist/domain/rel-types.js +7 -0
- package/dist/domain/rel-types.js.map +1 -1
- package/dist/domain/standard-sobjects.d.ts +20 -0
- package/dist/domain/standard-sobjects.d.ts.map +1 -0
- package/dist/domain/standard-sobjects.js +183 -0
- package/dist/domain/standard-sobjects.js.map +1 -0
- package/dist/embedding/cache.d.ts +41 -0
- package/dist/embedding/cache.d.ts.map +1 -0
- package/dist/embedding/cache.js +69 -0
- package/dist/embedding/cache.js.map +1 -0
- package/dist/embedding/embed.d.ts +59 -0
- package/dist/embedding/embed.d.ts.map +1 -1
- package/dist/embedding/embed.js +207 -29
- package/dist/embedding/embed.js.map +1 -1
- package/dist/embedding/index.d.ts +7 -0
- package/dist/embedding/index.d.ts.map +1 -1
- package/dist/embedding/index.js +4 -0
- package/dist/embedding/index.js.map +1 -1
- package/dist/embedding/pool.d.ts +66 -6
- package/dist/embedding/pool.d.ts.map +1 -1
- package/dist/embedding/pool.js +156 -6
- package/dist/embedding/pool.js.map +1 -1
- package/dist/embedding/queue.d.ts +166 -9
- package/dist/embedding/queue.d.ts.map +1 -1
- package/dist/embedding/queue.js +397 -39
- package/dist/embedding/queue.js.map +1 -1
- package/dist/embedding/tokenize.d.ts +18 -0
- package/dist/embedding/tokenize.d.ts.map +1 -0
- package/dist/embedding/tokenize.js +43 -0
- package/dist/embedding/tokenize.js.map +1 -0
- package/dist/embedding/worker.d.ts +0 -5
- package/dist/embedding/worker.d.ts.map +1 -1
- package/dist/embedding/worker.js +27 -4
- package/dist/embedding/worker.js.map +1 -1
- package/dist/extractors/filesystem/index.d.ts +21 -1
- package/dist/extractors/filesystem/index.d.ts.map +1 -1
- package/dist/extractors/filesystem/index.js +195 -38
- package/dist/extractors/filesystem/index.js.map +1 -1
- package/dist/extractors/filesystem/vlocity-datapacks.d.ts +27 -0
- package/dist/extractors/filesystem/vlocity-datapacks.d.ts.map +1 -0
- package/dist/extractors/filesystem/vlocity-datapacks.js +201 -0
- package/dist/extractors/filesystem/vlocity-datapacks.js.map +1 -0
- package/dist/extractors/interfaces/metadata-source.d.ts +40 -0
- package/dist/extractors/interfaces/metadata-source.d.ts.map +1 -1
- package/dist/extractors/live-org/bulk-retrieve.d.ts +206 -1
- package/dist/extractors/live-org/bulk-retrieve.d.ts.map +1 -1
- package/dist/extractors/live-org/bulk-retrieve.js +706 -94
- package/dist/extractors/live-org/bulk-retrieve.js.map +1 -1
- package/dist/extractors/live-org/capabilities.d.ts +15 -0
- package/dist/extractors/live-org/capabilities.d.ts.map +1 -1
- package/dist/extractors/live-org/capabilities.js +54 -9
- package/dist/extractors/live-org/capabilities.js.map +1 -1
- package/dist/extractors/live-org/extractors/apex.d.ts +12 -2
- package/dist/extractors/live-org/extractors/apex.d.ts.map +1 -1
- package/dist/extractors/live-org/extractors/apex.js +171 -75
- package/dist/extractors/live-org/extractors/apex.js.map +1 -1
- package/dist/extractors/live-org/extractors/custom-settings.d.ts +4 -0
- package/dist/extractors/live-org/extractors/custom-settings.d.ts.map +1 -0
- package/dist/extractors/live-org/extractors/custom-settings.js +76 -0
- package/dist/extractors/live-org/extractors/custom-settings.js.map +1 -0
- package/dist/extractors/live-org/extractors/flow.d.ts +2 -1
- package/dist/extractors/live-org/extractors/flow.d.ts.map +1 -1
- package/dist/extractors/live-org/extractors/flow.js +12 -3
- package/dist/extractors/live-org/extractors/flow.js.map +1 -1
- package/dist/extractors/live-org/extractors/generic-metadata.d.ts +23 -3
- package/dist/extractors/live-org/extractors/generic-metadata.d.ts.map +1 -1
- package/dist/extractors/live-org/extractors/generic-metadata.js +110 -25
- package/dist/extractors/live-org/extractors/generic-metadata.js.map +1 -1
- package/dist/extractors/live-org/extractors/generic-sobject.js.map +1 -1
- package/dist/extractors/live-org/extractors/integration.d.ts +2 -1
- package/dist/extractors/live-org/extractors/integration.d.ts.map +1 -1
- package/dist/extractors/live-org/extractors/integration.js +14 -5
- package/dist/extractors/live-org/extractors/integration.js.map +1 -1
- package/dist/extractors/live-org/extractors/lwc.d.ts +2 -1
- package/dist/extractors/live-org/extractors/lwc.d.ts.map +1 -1
- package/dist/extractors/live-org/extractors/lwc.js +95 -47
- package/dist/extractors/live-org/extractors/lwc.js.map +1 -1
- package/dist/extractors/live-org/extractors/mcd-baseline.d.ts.map +1 -1
- package/dist/extractors/live-org/extractors/mcd-baseline.js +2 -4
- package/dist/extractors/live-org/extractors/mcd-baseline.js.map +1 -1
- package/dist/extractors/live-org/extractors/object.d.ts +2 -11
- package/dist/extractors/live-org/extractors/object.d.ts.map +1 -1
- package/dist/extractors/live-org/extractors/object.js +127 -226
- package/dist/extractors/live-org/extractors/object.js.map +1 -1
- package/dist/extractors/live-org/extractors/omnistudio-retrieve.d.ts.map +1 -1
- package/dist/extractors/live-org/extractors/omnistudio-retrieve.js.map +1 -1
- package/dist/extractors/live-org/extractors/omnistudio.d.ts +11 -1
- package/dist/extractors/live-org/extractors/omnistudio.d.ts.map +1 -1
- package/dist/extractors/live-org/extractors/omnistudio.js +104 -16
- package/dist/extractors/live-org/extractors/omnistudio.js.map +1 -1
- package/dist/extractors/live-org/extractors/security.d.ts +12 -1
- package/dist/extractors/live-org/extractors/security.d.ts.map +1 -1
- package/dist/extractors/live-org/extractors/security.js +29 -7
- package/dist/extractors/live-org/extractors/security.js.map +1 -1
- package/dist/extractors/live-org/rate-limit.d.ts +48 -16
- package/dist/extractors/live-org/rate-limit.d.ts.map +1 -1
- package/dist/extractors/live-org/rate-limit.js +235 -40
- package/dist/extractors/live-org/rate-limit.js.map +1 -1
- package/dist/extractors/live-org/retry.d.ts +136 -0
- package/dist/extractors/live-org/retry.d.ts.map +1 -0
- package/dist/extractors/live-org/retry.js +416 -0
- package/dist/extractors/live-org/retry.js.map +1 -0
- package/dist/extractors/live-org/vlocity/runner.d.ts +2 -1
- package/dist/extractors/live-org/vlocity/runner.d.ts.map +1 -1
- package/dist/extractors/live-org/vlocity/runner.js +124 -72
- package/dist/extractors/live-org/vlocity/runner.js.map +1 -1
- package/dist/extractors/vlocity-shared/row.d.ts +19 -0
- package/dist/extractors/vlocity-shared/row.d.ts.map +1 -0
- package/dist/extractors/vlocity-shared/row.js +69 -0
- package/dist/extractors/vlocity-shared/row.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/ingest/delta.d.ts +70 -0
- package/dist/ingest/delta.d.ts.map +1 -0
- package/dist/ingest/delta.js +534 -0
- package/dist/ingest/delta.js.map +1 -0
- package/dist/ingest/detect-deletions-guard.d.ts +79 -0
- package/dist/ingest/detect-deletions-guard.d.ts.map +1 -0
- package/dist/ingest/detect-deletions-guard.js +122 -0
- package/dist/ingest/detect-deletions-guard.js.map +1 -0
- package/dist/ingest/index.d.ts +8 -1
- package/dist/ingest/index.d.ts.map +1 -1
- package/dist/ingest/index.js +4 -0
- package/dist/ingest/index.js.map +1 -1
- package/dist/ingest/live-ingest.d.ts +180 -0
- package/dist/ingest/live-ingest.d.ts.map +1 -1
- package/dist/ingest/live-ingest.js +1617 -625
- package/dist/ingest/live-ingest.js.map +1 -1
- package/dist/ingest/lock.d.ts +47 -0
- package/dist/ingest/lock.d.ts.map +1 -0
- package/dist/ingest/lock.js +222 -0
- package/dist/ingest/lock.js.map +1 -0
- package/dist/ingest/member-rows.d.ts +88 -0
- package/dist/ingest/member-rows.d.ts.map +1 -0
- package/dist/ingest/member-rows.js +152 -0
- package/dist/ingest/member-rows.js.map +1 -0
- package/dist/ingest/offline-org.d.ts +8 -0
- package/dist/ingest/offline-org.d.ts.map +1 -0
- package/dist/ingest/offline-org.js +41 -0
- package/dist/ingest/offline-org.js.map +1 -0
- package/dist/ingest/parse-member.d.ts +53 -0
- package/dist/ingest/parse-member.d.ts.map +1 -0
- package/dist/ingest/parse-member.js +276 -0
- package/dist/ingest/parse-member.js.map +1 -0
- package/dist/ingest/parse-pool.d.ts +139 -0
- package/dist/ingest/parse-pool.d.ts.map +1 -0
- package/dist/ingest/parse-pool.js +593 -0
- package/dist/ingest/parse-pool.js.map +1 -0
- package/dist/ingest/parse-worker.d.ts +3 -0
- package/dist/ingest/parse-worker.d.ts.map +1 -0
- package/dist/ingest/parse-worker.js +100 -0
- package/dist/ingest/parse-worker.js.map +1 -0
- package/dist/ingest/profile.d.ts +27 -0
- package/dist/ingest/profile.d.ts.map +1 -0
- package/dist/ingest/profile.js +48 -0
- package/dist/ingest/profile.js.map +1 -0
- package/dist/ingest/resolver-gates.d.ts +12 -0
- package/dist/ingest/resolver-gates.d.ts.map +1 -0
- package/dist/ingest/resolver-gates.js +53 -0
- package/dist/ingest/resolver-gates.js.map +1 -0
- package/dist/parsers/apex/arity-resolver.d.ts +16 -3
- package/dist/parsers/apex/arity-resolver.d.ts.map +1 -1
- package/dist/parsers/apex/arity-resolver.js +31 -2
- package/dist/parsers/apex/arity-resolver.js.map +1 -1
- package/dist/parsers/apex/ast-extractor.d.ts +4 -0
- package/dist/parsers/apex/ast-extractor.d.ts.map +1 -1
- package/dist/parsers/apex/ast-extractor.js +128 -29
- package/dist/parsers/apex/ast-extractor.js.map +1 -1
- package/dist/parsers/apex/class-refs.d.ts +78 -0
- package/dist/parsers/apex/class-refs.d.ts.map +1 -0
- package/dist/parsers/apex/class-refs.js +465 -0
- package/dist/parsers/apex/class-refs.js.map +1 -0
- package/dist/parsers/apex/class.d.ts +158 -0
- package/dist/parsers/apex/class.d.ts.map +1 -1
- package/dist/parsers/apex/class.js +1073 -172
- package/dist/parsers/apex/class.js.map +1 -1
- package/dist/parsers/apex/dynamic-soql.d.ts +52 -0
- package/dist/parsers/apex/dynamic-soql.d.ts.map +1 -0
- package/dist/parsers/apex/dynamic-soql.js +427 -0
- package/dist/parsers/apex/dynamic-soql.js.map +1 -0
- package/dist/parsers/apex/field-access.d.ts +65 -0
- package/dist/parsers/apex/field-access.d.ts.map +1 -0
- package/dist/parsers/apex/field-access.js +438 -0
- package/dist/parsers/apex/field-access.js.map +1 -0
- package/dist/parsers/apex/interface-ref-resolver.d.ts +32 -0
- package/dist/parsers/apex/interface-ref-resolver.d.ts.map +1 -0
- package/dist/parsers/apex/interface-ref-resolver.js +64 -0
- package/dist/parsers/apex/interface-ref-resolver.js.map +1 -0
- package/dist/parsers/apex/name-refs.d.ts +83 -0
- package/dist/parsers/apex/name-refs.d.ts.map +1 -0
- package/dist/parsers/apex/name-refs.js +465 -0
- package/dist/parsers/apex/name-refs.js.map +1 -0
- package/dist/parsers/apex/page-refs.d.ts +11 -0
- package/dist/parsers/apex/page-refs.d.ts.map +1 -0
- package/dist/parsers/apex/page-refs.js +34 -0
- package/dist/parsers/apex/page-refs.js.map +1 -0
- package/dist/parsers/apex/ref-resolver.d.ts +58 -0
- package/dist/parsers/apex/ref-resolver.d.ts.map +1 -0
- package/dist/parsers/apex/ref-resolver.js +348 -0
- package/dist/parsers/apex/ref-resolver.js.map +1 -0
- package/dist/parsers/apex/soql-fields.d.ts +75 -0
- package/dist/parsers/apex/soql-fields.d.ts.map +1 -0
- package/dist/parsers/apex/soql-fields.js +338 -0
- package/dist/parsers/apex/soql-fields.js.map +1 -0
- package/dist/parsers/apex/trigger.d.ts.map +1 -1
- package/dist/parsers/apex/trigger.js +100 -2
- package/dist/parsers/apex/trigger.js.map +1 -1
- package/dist/parsers/apex/vlocity-calls.d.ts +25 -0
- package/dist/parsers/apex/vlocity-calls.d.ts.map +1 -0
- package/dist/parsers/apex/vlocity-calls.js +225 -0
- package/dist/parsers/apex/vlocity-calls.js.map +1 -0
- package/dist/parsers/aura/bundle.d.ts +39 -0
- package/dist/parsers/aura/bundle.d.ts.map +1 -0
- package/dist/parsers/aura/bundle.js +269 -0
- package/dist/parsers/aura/bundle.js.map +1 -0
- package/dist/parsers/aura/embed-resolver.d.ts +17 -0
- package/dist/parsers/aura/embed-resolver.d.ts.map +1 -0
- package/dist/parsers/aura/embed-resolver.js +58 -0
- package/dist/parsers/aura/embed-resolver.js.map +1 -0
- package/dist/parsers/aura/index.d.ts +4 -0
- package/dist/parsers/aura/index.d.ts.map +1 -0
- package/dist/parsers/aura/index.js +6 -0
- package/dist/parsers/aura/index.js.map +1 -0
- package/dist/parsers/common.d.ts +9 -0
- package/dist/parsers/common.d.ts.map +1 -1
- package/dist/parsers/common.js +17 -0
- package/dist/parsers/common.js.map +1 -1
- package/dist/parsers/cross-flavor-resolver.js +8 -2
- package/dist/parsers/cross-flavor-resolver.js.map +1 -1
- package/dist/parsers/flow/invocable-resolver.d.ts.map +1 -1
- package/dist/parsers/flow/invocable-resolver.js +2 -0
- package/dist/parsers/flow/invocable-resolver.js.map +1 -1
- package/dist/parsers/flow/parser.d.ts +9 -0
- package/dist/parsers/flow/parser.d.ts.map +1 -1
- package/dist/parsers/flow/parser.js +327 -2
- package/dist/parsers/flow/parser.js.map +1 -1
- package/dist/parsers/flow/relationship-resolver.d.ts +34 -0
- package/dist/parsers/flow/relationship-resolver.d.ts.map +1 -0
- package/dist/parsers/flow/relationship-resolver.js +199 -0
- package/dist/parsers/flow/relationship-resolver.js.map +1 -0
- package/dist/parsers/generic/opaque-node-parser.d.ts +5 -1
- package/dist/parsers/generic/opaque-node-parser.d.ts.map +1 -1
- package/dist/parsers/generic/opaque-node-parser.js +37 -5
- package/dist/parsers/generic/opaque-node-parser.js.map +1 -1
- package/dist/parsers/generic/reflection-walker.d.ts +7 -0
- package/dist/parsers/generic/reflection-walker.d.ts.map +1 -1
- package/dist/parsers/generic/reflection-walker.js +38 -13
- package/dist/parsers/generic/reflection-walker.js.map +1 -1
- package/dist/parsers/hash-salts.d.ts +72 -0
- package/dist/parsers/hash-salts.d.ts.map +1 -0
- package/dist/parsers/hash-salts.js +91 -0
- package/dist/parsers/hash-salts.js.map +1 -0
- package/dist/parsers/index.d.ts +2 -0
- package/dist/parsers/index.d.ts.map +1 -1
- package/dist/parsers/index.js +2 -0
- package/dist/parsers/index.js.map +1 -1
- package/dist/parsers/lwc/bundle.d.ts +1 -0
- package/dist/parsers/lwc/bundle.d.ts.map +1 -1
- package/dist/parsers/lwc/bundle.js +37 -2
- package/dist/parsers/lwc/bundle.js.map +1 -1
- package/dist/parsers/lwc/field-key-resolver.d.ts +32 -0
- package/dist/parsers/lwc/field-key-resolver.d.ts.map +1 -0
- package/dist/parsers/lwc/field-key-resolver.js +92 -0
- package/dist/parsers/lwc/field-key-resolver.js.map +1 -0
- package/dist/parsers/lwc/html-visitor.d.ts +1 -1
- package/dist/parsers/lwc/html-visitor.d.ts.map +1 -1
- package/dist/parsers/lwc/html-visitor.js +81 -3
- package/dist/parsers/lwc/html-visitor.js.map +1 -1
- package/dist/parsers/lwc/js-visitor.d.ts +16 -0
- package/dist/parsers/lwc/js-visitor.d.ts.map +1 -1
- package/dist/parsers/lwc/js-visitor.js +354 -12
- package/dist/parsers/lwc/js-visitor.js.map +1 -1
- package/dist/parsers/lwc/omnidef.d.ts +10 -0
- package/dist/parsers/lwc/omnidef.d.ts.map +1 -0
- package/dist/parsers/lwc/omnidef.js +127 -0
- package/dist/parsers/lwc/omnidef.js.map +1 -0
- package/dist/parsers/lwc/xml-meta.d.ts.map +1 -1
- package/dist/parsers/lwc/xml-meta.js +16 -1
- package/dist/parsers/lwc/xml-meta.js.map +1 -1
- package/dist/parsers/object/formula-hop-resolver.d.ts +33 -0
- package/dist/parsers/object/formula-hop-resolver.d.ts.map +1 -0
- package/dist/parsers/object/formula-hop-resolver.js +60 -0
- package/dist/parsers/object/formula-hop-resolver.js.map +1 -0
- package/dist/parsers/object/formula-refs.d.ts +40 -0
- package/dist/parsers/object/formula-refs.d.ts.map +1 -0
- package/dist/parsers/object/formula-refs.js +129 -0
- package/dist/parsers/object/formula-refs.js.map +1 -0
- package/dist/parsers/object/object.d.ts +3 -0
- package/dist/parsers/object/object.d.ts.map +1 -1
- package/dist/parsers/object/object.js +54 -57
- package/dist/parsers/object/object.js.map +1 -1
- package/dist/parsers/omnistudio/data-transform.d.ts.map +1 -1
- package/dist/parsers/omnistudio/data-transform.js +40 -7
- package/dist/parsers/omnistudio/data-transform.js.map +1 -1
- package/dist/parsers/omnistudio/integration-procedure.d.ts.map +1 -1
- package/dist/parsers/omnistudio/integration-procedure.js +6 -6
- package/dist/parsers/omnistudio/integration-procedure.js.map +1 -1
- package/dist/parsers/omnistudio/overlap-detector.d.ts.map +1 -1
- package/dist/parsers/omnistudio/overlap-detector.js +29 -11
- package/dist/parsers/omnistudio/overlap-detector.js.map +1 -1
- package/dist/parsers/omnistudio/process.d.ts.map +1 -1
- package/dist/parsers/omnistudio/process.js +6 -6
- package/dist/parsers/omnistudio/process.js.map +1 -1
- package/dist/parsers/omnistudio/remote-refs.d.ts +11 -0
- package/dist/parsers/omnistudio/remote-refs.d.ts.map +1 -0
- package/dist/parsers/omnistudio/remote-refs.js +17 -0
- package/dist/parsers/omnistudio/remote-refs.js.map +1 -0
- package/dist/parsers/omnistudio/ui-card.d.ts.map +1 -1
- package/dist/parsers/omnistudio/ui-card.js +6 -6
- package/dist/parsers/omnistudio/ui-card.js.map +1 -1
- package/dist/parsers/resolver-prune.d.ts +14 -0
- package/dist/parsers/resolver-prune.d.ts.map +1 -0
- package/dist/parsers/resolver-prune.js +29 -0
- package/dist/parsers/resolver-prune.js.map +1 -0
- package/dist/parsers/rules/_engine.d.ts.map +1 -1
- package/dist/parsers/rules/_engine.js +4 -0
- package/dist/parsers/rules/_engine.js.map +1 -1
- package/dist/parsers/rules/_engine.ts +3 -0
- package/dist/parsers/rules/_loader.d.ts +8 -0
- package/dist/parsers/rules/_loader.d.ts.map +1 -1
- package/dist/parsers/rules/_loader.js +17 -0
- package/dist/parsers/rules/_loader.js.map +1 -1
- package/dist/parsers/rules/_loader.ts +19 -0
- package/dist/parsers/rules/_schema.d.ts +12 -12
- package/dist/parsers/rules/_selectors.d.ts +1 -0
- package/dist/parsers/rules/_selectors.d.ts.map +1 -1
- package/dist/parsers/rules/_selectors.js +6 -0
- package/dist/parsers/rules/_selectors.js.map +1 -1
- package/dist/parsers/rules/_selectors.ts +6 -0
- package/dist/parsers/rules/apex-component.yml +49 -4
- package/dist/parsers/rules/apex-page.yml +49 -4
- package/dist/parsers/rules/approval-process.yml +25 -5
- package/dist/parsers/rules/custom-metadata-type.yml +14 -7
- package/dist/parsers/rules/custom-setting.yml +24 -0
- package/dist/parsers/rules/custom-tab.yml +41 -0
- package/dist/parsers/rules/lightning-page.yml +4 -2
- package/dist/parsers/rules/workflow.yml +30 -10
- package/dist/parsers/vf/refs.d.ts +39 -0
- package/dist/parsers/vf/refs.d.ts.map +1 -0
- package/dist/parsers/vf/refs.js +152 -0
- package/dist/parsers/vf/refs.js.map +1 -0
- package/dist/parsers/vlocity/apex-resolver.d.ts +21 -0
- package/dist/parsers/vlocity/apex-resolver.d.ts.map +1 -0
- package/dist/parsers/vlocity/apex-resolver.js +115 -0
- package/dist/parsers/vlocity/apex-resolver.js.map +1 -0
- package/dist/parsers/vlocity/common.d.ts +72 -1
- package/dist/parsers/vlocity/common.d.ts.map +1 -1
- package/dist/parsers/vlocity/common.js +332 -2
- package/dist/parsers/vlocity/common.js.map +1 -1
- package/dist/parsers/vlocity/data-raptor.d.ts +9 -0
- package/dist/parsers/vlocity/data-raptor.d.ts.map +1 -1
- package/dist/parsers/vlocity/data-raptor.js +148 -17
- package/dist/parsers/vlocity/data-raptor.js.map +1 -1
- package/dist/parsers/vlocity/datapack-refs.d.ts +22 -0
- package/dist/parsers/vlocity/datapack-refs.d.ts.map +1 -0
- package/dist/parsers/vlocity/datapack-refs.js +126 -0
- package/dist/parsers/vlocity/datapack-refs.js.map +1 -0
- package/dist/parsers/vlocity/dr-relationship-resolver.d.ts +26 -0
- package/dist/parsers/vlocity/dr-relationship-resolver.d.ts.map +1 -0
- package/dist/parsers/vlocity/dr-relationship-resolver.js +137 -0
- package/dist/parsers/vlocity/dr-relationship-resolver.js.map +1 -0
- package/dist/parsers/vlocity/integration-procedure.d.ts.map +1 -1
- package/dist/parsers/vlocity/integration-procedure.js +15 -48
- package/dist/parsers/vlocity/integration-procedure.js.map +1 -1
- package/dist/parsers/vlocity/name-match-resolver.d.ts +21 -0
- package/dist/parsers/vlocity/name-match-resolver.d.ts.map +1 -0
- package/dist/parsers/vlocity/name-match-resolver.js +97 -0
- package/dist/parsers/vlocity/name-match-resolver.js.map +1 -0
- package/dist/parsers/vlocity/omni-script.d.ts.map +1 -1
- package/dist/parsers/vlocity/omni-script.js +18 -32
- package/dist/parsers/vlocity/omni-script.js.map +1 -1
- package/dist/parsers/vlocity/vlocity-card.d.ts.map +1 -1
- package/dist/parsers/vlocity/vlocity-card.js +30 -36
- package/dist/parsers/vlocity/vlocity-card.js.map +1 -1
- package/dist/render/mermaid/dependency-graph.d.ts.map +1 -1
- package/dist/render/mermaid/dependency-graph.js +13 -1
- package/dist/render/mermaid/dependency-graph.js.map +1 -1
- package/dist/render/mermaid/diff.d.ts.map +1 -1
- package/dist/render/mermaid/diff.js +4 -2
- package/dist/render/mermaid/diff.js.map +1 -1
- package/dist/render/mermaid/shared.d.ts +7 -0
- package/dist/render/mermaid/shared.d.ts.map +1 -1
- package/dist/render/mermaid/shared.js +15 -6
- package/dist/render/mermaid/shared.js.map +1 -1
- package/dist/render/sarif.d.ts.map +1 -1
- package/dist/render/sarif.js +1 -3
- package/dist/render/sarif.js.map +1 -1
- package/dist/storage/index.d.ts +3 -3
- package/dist/storage/index.d.ts.map +1 -1
- package/dist/storage/index.js +2 -2
- package/dist/storage/index.js.map +1 -1
- package/dist/storage/interfaces.d.ts +212 -1
- package/dist/storage/interfaces.d.ts.map +1 -1
- package/dist/storage/sqlite/graph-store.d.ts +366 -3
- package/dist/storage/sqlite/graph-store.d.ts.map +1 -1
- package/dist/storage/sqlite/graph-store.js +1755 -68
- package/dist/storage/sqlite/graph-store.js.map +1 -1
- package/dist/storage/sqlite/migrations.d.ts +11 -0
- package/dist/storage/sqlite/migrations.d.ts.map +1 -1
- package/dist/storage/sqlite/migrations.js +245 -11
- package/dist/storage/sqlite/migrations.js.map +1 -1
- package/dist/storage/sqlite/rename-stability.d.ts +1 -1
- package/dist/storage/sqlite/rename-stability.d.ts.map +1 -1
- package/dist/storage/sqlite/rename-stability.js +2 -6
- package/dist/storage/sqlite/rename-stability.js.map +1 -1
- package/dist/storage/sqlite/snapshot-store.d.ts +79 -3
- package/dist/storage/sqlite/snapshot-store.d.ts.map +1 -1
- package/dist/storage/sqlite/snapshot-store.js +500 -100
- package/dist/storage/sqlite/snapshot-store.js.map +1 -1
- package/dist/storage/sqlite/snapshot-versions.d.ts +51 -0
- package/dist/storage/sqlite/snapshot-versions.d.ts.map +1 -0
- package/dist/storage/sqlite/snapshot-versions.js +209 -0
- package/dist/storage/sqlite/snapshot-versions.js.map +1 -0
- package/dist/storage/sqlite/vector-store.d.ts +12 -0
- package/dist/storage/sqlite/vector-store.d.ts.map +1 -1
- package/dist/storage/sqlite/vector-store.js +57 -14
- package/dist/storage/sqlite/vector-store.js.map +1 -1
- package/dist/util/process-owner.d.ts +35 -0
- package/dist/util/process-owner.d.ts.map +1 -0
- package/dist/util/process-owner.js +87 -0
- package/dist/util/process-owner.js.map +1 -0
- package/dist/util/worker-heap-cap.d.ts +10 -0
- package/dist/util/worker-heap-cap.d.ts.map +1 -0
- package/dist/util/worker-heap-cap.js +65 -0
- package/dist/util/worker-heap-cap.js.map +1 -0
- package/package.json +6 -7
- package/dist/analyze/__tests__/audit-graph.test.d.ts +0 -2
- package/dist/analyze/__tests__/audit-graph.test.d.ts.map +0 -1
- package/dist/analyze/__tests__/audit-graph.test.js +0 -122
- package/dist/analyze/__tests__/audit-graph.test.js.map +0 -1
- package/dist/analyze/__tests__/cross-org-two-store.test.d.ts +0 -2
- package/dist/analyze/__tests__/cross-org-two-store.test.d.ts.map +0 -1
- package/dist/analyze/__tests__/cross-org-two-store.test.js +0 -68
- package/dist/analyze/__tests__/cross-org-two-store.test.js.map +0 -1
- package/dist/analyze/__tests__/dead-code-confidence.test.d.ts +0 -2
- package/dist/analyze/__tests__/dead-code-confidence.test.d.ts.map +0 -1
- package/dist/analyze/__tests__/dead-code-confidence.test.js +0 -66
- package/dist/analyze/__tests__/dead-code-confidence.test.js.map +0 -1
- package/dist/analyze/__tests__/deployment-manifest.test.d.ts +0 -2
- package/dist/analyze/__tests__/deployment-manifest.test.d.ts.map +0 -1
- package/dist/analyze/__tests__/deployment-manifest.test.js +0 -66
- package/dist/analyze/__tests__/deployment-manifest.test.js.map +0 -1
- package/dist/analyze/__tests__/find-nodes.test.d.ts +0 -2
- package/dist/analyze/__tests__/find-nodes.test.d.ts.map +0 -1
- package/dist/analyze/__tests__/find-nodes.test.js +0 -129
- package/dist/analyze/__tests__/find-nodes.test.js.map +0 -1
- package/dist/analyze/__tests__/findings.test.d.ts +0 -2
- package/dist/analyze/__tests__/findings.test.d.ts.map +0 -1
- package/dist/analyze/__tests__/findings.test.js +0 -90
- package/dist/analyze/__tests__/findings.test.js.map +0 -1
- package/dist/analyze/__tests__/governor-risks-detection.test.d.ts +0 -2
- package/dist/analyze/__tests__/governor-risks-detection.test.d.ts.map +0 -1
- package/dist/analyze/__tests__/governor-risks-detection.test.js +0 -70
- package/dist/analyze/__tests__/governor-risks-detection.test.js.map +0 -1
- package/dist/analyze/__tests__/populate.test.d.ts +0 -2
- package/dist/analyze/__tests__/populate.test.d.ts.map +0 -1
- package/dist/analyze/__tests__/populate.test.js +0 -129
- package/dist/analyze/__tests__/populate.test.js.map +0 -1
- package/dist/analyze/__tests__/traversal-caps.test.d.ts +0 -2
- package/dist/analyze/__tests__/traversal-caps.test.d.ts.map +0 -1
- package/dist/analyze/__tests__/traversal-caps.test.js +0 -129
- package/dist/analyze/__tests__/traversal-caps.test.js.map +0 -1
- package/dist/analyze/__tests__/wip-impact.test.d.ts +0 -2
- package/dist/analyze/__tests__/wip-impact.test.d.ts.map +0 -1
- package/dist/analyze/__tests__/wip-impact.test.js +0 -134
- package/dist/analyze/__tests__/wip-impact.test.js.map +0 -1
- package/dist/embedding/__tests__/embed.test.d.ts +0 -2
- package/dist/embedding/__tests__/embed.test.d.ts.map +0 -1
- package/dist/embedding/__tests__/embed.test.js +0 -72
- package/dist/embedding/__tests__/embed.test.js.map +0 -1
- package/dist/embedding/__tests__/queue.test.d.ts +0 -2
- package/dist/embedding/__tests__/queue.test.d.ts.map +0 -1
- package/dist/embedding/__tests__/queue.test.js +0 -64
- package/dist/embedding/__tests__/queue.test.js.map +0 -1
- package/dist/extractors/__tests__/network-egress.test.d.ts +0 -2
- package/dist/extractors/__tests__/network-egress.test.d.ts.map +0 -1
- package/dist/extractors/__tests__/network-egress.test.js +0 -32
- package/dist/extractors/__tests__/network-egress.test.js.map +0 -1
- package/dist/extractors/filesystem/__tests__/filesystem.test.d.ts +0 -2
- package/dist/extractors/filesystem/__tests__/filesystem.test.d.ts.map +0 -1
- package/dist/extractors/filesystem/__tests__/filesystem.test.js +0 -96
- package/dist/extractors/filesystem/__tests__/filesystem.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/_jsforce-mock.d.ts +0 -38
- package/dist/extractors/live-org/__tests__/_jsforce-mock.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/_jsforce-mock.js +0 -96
- package/dist/extractors/live-org/__tests__/_jsforce-mock.js.map +0 -1
- package/dist/extractors/live-org/__tests__/auth.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/auth.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/auth.test.js +0 -85
- package/dist/extractors/live-org/__tests__/auth.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/bulk-retrieve.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/bulk-retrieve.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/bulk-retrieve.test.js +0 -150
- package/dist/extractors/live-org/__tests__/bulk-retrieve.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/capabilities.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/capabilities.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/capabilities.test.js +0 -54
- package/dist/extractors/live-org/__tests__/capabilities.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/discovery.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/discovery.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/discovery.test.js +0 -55
- package/dist/extractors/live-org/__tests__/discovery.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/dispatch.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/dispatch.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/dispatch.test.js +0 -68
- package/dist/extractors/live-org/__tests__/dispatch.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/extractor-enrichment.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/extractor-enrichment.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/extractor-enrichment.test.js +0 -312
- package/dist/extractors/live-org/__tests__/extractor-enrichment.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/liveness.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/liveness.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/liveness.test.js +0 -148
- package/dist/extractors/live-org/__tests__/liveness.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/mcd-baseline.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/mcd-baseline.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/mcd-baseline.test.js +0 -174
- package/dist/extractors/live-org/__tests__/mcd-baseline.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/metadata-read-batch.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/metadata-read-batch.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/metadata-read-batch.test.js +0 -75
- package/dist/extractors/live-org/__tests__/metadata-read-batch.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/omnistudio-retrieve.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/omnistudio-retrieve.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/omnistudio-retrieve.test.js +0 -135
- package/dist/extractors/live-org/__tests__/omnistudio-retrieve.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/pools.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/pools.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/pools.test.js +0 -75
- package/dist/extractors/live-org/__tests__/pools.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/rate-limit.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/rate-limit.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/rate-limit.test.js +0 -78
- package/dist/extractors/live-org/__tests__/rate-limit.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/read-only-proxy.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/read-only-proxy.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/read-only-proxy.test.js +0 -376
- package/dist/extractors/live-org/__tests__/read-only-proxy.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/soql-rebatch.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/soql-rebatch.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/soql-rebatch.test.js +0 -118
- package/dist/extractors/live-org/__tests__/soql-rebatch.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/source-member.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/source-member.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/source-member.test.js +0 -84
- package/dist/extractors/live-org/__tests__/source-member.test.js.map +0 -1
- package/dist/extractors/live-org/__tests__/vlocity-runner.test.d.ts +0 -2
- package/dist/extractors/live-org/__tests__/vlocity-runner.test.d.ts.map +0 -1
- package/dist/extractors/live-org/__tests__/vlocity-runner.test.js +0 -115
- package/dist/extractors/live-org/__tests__/vlocity-runner.test.js.map +0 -1
- package/dist/ingest/__tests__/full-sync-deletion.test.d.ts +0 -2
- package/dist/ingest/__tests__/full-sync-deletion.test.d.ts.map +0 -1
- package/dist/ingest/__tests__/full-sync-deletion.test.js +0 -200
- package/dist/ingest/__tests__/full-sync-deletion.test.js.map +0 -1
- package/dist/ingest/__tests__/incremental-deletion.test.d.ts +0 -2
- package/dist/ingest/__tests__/incremental-deletion.test.d.ts.map +0 -1
- package/dist/ingest/__tests__/incremental-deletion.test.js +0 -92
- package/dist/ingest/__tests__/incremental-deletion.test.js.map +0 -1
- package/dist/ingest/__tests__/live-ingest.test.d.ts +0 -2
- package/dist/ingest/__tests__/live-ingest.test.d.ts.map +0 -1
- package/dist/ingest/__tests__/live-ingest.test.js +0 -155
- package/dist/ingest/__tests__/live-ingest.test.js.map +0 -1
- package/dist/ingest/__tests__/multi-org-orchestrator.test.d.ts +0 -2
- package/dist/ingest/__tests__/multi-org-orchestrator.test.d.ts.map +0 -1
- package/dist/ingest/__tests__/multi-org-orchestrator.test.js +0 -108
- package/dist/ingest/__tests__/multi-org-orchestrator.test.js.map +0 -1
- package/dist/ingest/__tests__/read-only-wrapping.test.d.ts +0 -2
- package/dist/ingest/__tests__/read-only-wrapping.test.d.ts.map +0 -1
- package/dist/ingest/__tests__/read-only-wrapping.test.js +0 -62
- package/dist/ingest/__tests__/read-only-wrapping.test.js.map +0 -1
- package/dist/parsers/__tests__/_harness.d.ts +0 -13
- package/dist/parsers/__tests__/_harness.d.ts.map +0 -1
- package/dist/parsers/__tests__/_harness.js +0 -84
- package/dist/parsers/__tests__/_harness.js.map +0 -1
- package/dist/parsers/__tests__/apex.golden.test.d.ts +0 -2
- package/dist/parsers/__tests__/apex.golden.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/apex.golden.test.js +0 -28
- package/dist/parsers/__tests__/apex.golden.test.js.map +0 -1
- package/dist/parsers/__tests__/cross-flavor.test.d.ts +0 -2
- package/dist/parsers/__tests__/cross-flavor.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/cross-flavor.test.js +0 -56
- package/dist/parsers/__tests__/cross-flavor.test.js.map +0 -1
- package/dist/parsers/__tests__/edge-provenance.test.d.ts +0 -2
- package/dist/parsers/__tests__/edge-provenance.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/edge-provenance.test.js +0 -44
- package/dist/parsers/__tests__/edge-provenance.test.js.map +0 -1
- package/dist/parsers/__tests__/flow.golden.test.d.ts +0 -2
- package/dist/parsers/__tests__/flow.golden.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/flow.golden.test.js +0 -17
- package/dist/parsers/__tests__/flow.golden.test.js.map +0 -1
- package/dist/parsers/__tests__/integration.golden.test.d.ts +0 -2
- package/dist/parsers/__tests__/integration.golden.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/integration.golden.test.js +0 -31
- package/dist/parsers/__tests__/integration.golden.test.js.map +0 -1
- package/dist/parsers/__tests__/long-tail.golden.test.d.ts +0 -2
- package/dist/parsers/__tests__/long-tail.golden.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/long-tail.golden.test.js +0 -115
- package/dist/parsers/__tests__/long-tail.golden.test.js.map +0 -1
- package/dist/parsers/__tests__/lwc-bindings.test.d.ts +0 -2
- package/dist/parsers/__tests__/lwc-bindings.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/lwc-bindings.test.js +0 -146
- package/dist/parsers/__tests__/lwc-bindings.test.js.map +0 -1
- package/dist/parsers/__tests__/lwc.golden.test.d.ts +0 -2
- package/dist/parsers/__tests__/lwc.golden.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/lwc.golden.test.js +0 -16
- package/dist/parsers/__tests__/lwc.golden.test.js.map +0 -1
- package/dist/parsers/__tests__/mcd-gap-fills.test.d.ts +0 -2
- package/dist/parsers/__tests__/mcd-gap-fills.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/mcd-gap-fills.test.js +0 -75
- package/dist/parsers/__tests__/mcd-gap-fills.test.js.map +0 -1
- package/dist/parsers/__tests__/object-inline-fields.test.d.ts +0 -2
- package/dist/parsers/__tests__/object-inline-fields.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/object-inline-fields.test.js +0 -115
- package/dist/parsers/__tests__/object-inline-fields.test.js.map +0 -1
- package/dist/parsers/__tests__/object.golden.test.d.ts +0 -2
- package/dist/parsers/__tests__/object.golden.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/object.golden.test.js +0 -33
- package/dist/parsers/__tests__/object.golden.test.js.map +0 -1
- package/dist/parsers/__tests__/omnistudio-overlap.test.d.ts +0 -2
- package/dist/parsers/__tests__/omnistudio-overlap.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/omnistudio-overlap.test.js +0 -163
- package/dist/parsers/__tests__/omnistudio-overlap.test.js.map +0 -1
- package/dist/parsers/__tests__/omnistudio.golden.test.d.ts +0 -2
- package/dist/parsers/__tests__/omnistudio.golden.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/omnistudio.golden.test.js +0 -54
- package/dist/parsers/__tests__/omnistudio.golden.test.js.map +0 -1
- package/dist/parsers/__tests__/reflection-walker.test.d.ts +0 -2
- package/dist/parsers/__tests__/reflection-walker.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/reflection-walker.test.js +0 -162
- package/dist/parsers/__tests__/reflection-walker.test.js.map +0 -1
- package/dist/parsers/__tests__/registry.test.d.ts +0 -2
- package/dist/parsers/__tests__/registry.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/registry.test.js +0 -23
- package/dist/parsers/__tests__/registry.test.js.map +0 -1
- package/dist/parsers/__tests__/security.golden.test.d.ts +0 -2
- package/dist/parsers/__tests__/security.golden.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/security.golden.test.js +0 -31
- package/dist/parsers/__tests__/security.golden.test.js.map +0 -1
- package/dist/parsers/__tests__/vlocity.golden.test.d.ts +0 -2
- package/dist/parsers/__tests__/vlocity.golden.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/vlocity.golden.test.js +0 -109
- package/dist/parsers/__tests__/vlocity.golden.test.js.map +0 -1
- package/dist/parsers/__tests__/worker-pool.test.d.ts +0 -2
- package/dist/parsers/__tests__/worker-pool.test.d.ts.map +0 -1
- package/dist/parsers/__tests__/worker-pool.test.js +0 -48
- package/dist/parsers/__tests__/worker-pool.test.js.map +0 -1
- package/dist/parsers/apex/__tests__/arity-resolver.test.d.ts +0 -2
- package/dist/parsers/apex/__tests__/arity-resolver.test.d.ts.map +0 -1
- package/dist/parsers/apex/__tests__/arity-resolver.test.js +0 -135
- package/dist/parsers/apex/__tests__/arity-resolver.test.js.map +0 -1
- package/dist/parsers/apex/__tests__/ast-extractor.test.d.ts +0 -2
- package/dist/parsers/apex/__tests__/ast-extractor.test.d.ts.map +0 -1
- package/dist/parsers/apex/__tests__/ast-extractor.test.js +0 -146
- package/dist/parsers/apex/__tests__/ast-extractor.test.js.map +0 -1
- package/dist/parsers/apex/__tests__/is-test-attribute.test.d.ts +0 -2
- package/dist/parsers/apex/__tests__/is-test-attribute.test.d.ts.map +0 -1
- package/dist/parsers/apex/__tests__/is-test-attribute.test.js +0 -54
- package/dist/parsers/apex/__tests__/is-test-attribute.test.js.map +0 -1
- package/dist/parsers/apex/__tests__/regex-arg-count.test.d.ts +0 -2
- package/dist/parsers/apex/__tests__/regex-arg-count.test.d.ts.map +0 -1
- package/dist/parsers/apex/__tests__/regex-arg-count.test.js +0 -82
- package/dist/parsers/apex/__tests__/regex-arg-count.test.js.map +0 -1
- package/dist/parsers/flow/__tests__/invocable-resolver.test.d.ts +0 -2
- package/dist/parsers/flow/__tests__/invocable-resolver.test.d.ts.map +0 -1
- package/dist/parsers/flow/__tests__/invocable-resolver.test.js +0 -91
- package/dist/parsers/flow/__tests__/invocable-resolver.test.js.map +0 -1
- package/dist/parsers/rules/__tests__/applies-to.test.d.ts +0 -2
- package/dist/parsers/rules/__tests__/applies-to.test.d.ts.map +0 -1
- package/dist/parsers/rules/__tests__/applies-to.test.js +0 -31
- package/dist/parsers/rules/__tests__/applies-to.test.js.map +0 -1
- package/dist/parsers/rules/__tests__/applies-to.test.ts +0 -40
- package/dist/parsers/rules/__tests__/engine.test.d.ts +0 -2
- package/dist/parsers/rules/__tests__/engine.test.d.ts.map +0 -1
- package/dist/parsers/rules/__tests__/engine.test.js +0 -127
- package/dist/parsers/rules/__tests__/engine.test.js.map +0 -1
- package/dist/parsers/rules/__tests__/engine.test.ts +0 -138
- package/dist/parsers/rules/__tests__/loader.test.d.ts +0 -2
- package/dist/parsers/rules/__tests__/loader.test.d.ts.map +0 -1
- package/dist/parsers/rules/__tests__/loader.test.js +0 -31
- package/dist/parsers/rules/__tests__/loader.test.js.map +0 -1
- package/dist/parsers/rules/__tests__/loader.test.ts +0 -37
- package/dist/parsers/rules/__tests__/schema.test.d.ts +0 -2
- package/dist/parsers/rules/__tests__/schema.test.d.ts.map +0 -1
- package/dist/parsers/rules/__tests__/schema.test.js +0 -33
- package/dist/parsers/rules/__tests__/schema.test.js.map +0 -1
- package/dist/parsers/rules/__tests__/schema.test.ts +0 -37
- package/dist/parsers/rules/__tests__/selectors.test.d.ts +0 -2
- package/dist/parsers/rules/__tests__/selectors.test.d.ts.map +0 -1
- package/dist/parsers/rules/__tests__/selectors.test.js +0 -51
- package/dist/parsers/rules/__tests__/selectors.test.js.map +0 -1
- package/dist/parsers/rules/__tests__/selectors.test.ts +0 -62
- package/dist/render/__tests__/sarif.test.d.ts +0 -2
- package/dist/render/__tests__/sarif.test.d.ts.map +0 -1
- package/dist/render/__tests__/sarif.test.js +0 -129
- package/dist/render/__tests__/sarif.test.js.map +0 -1
- package/dist/render/mermaid/__tests__/mermaid.test.d.ts +0 -2
- package/dist/render/mermaid/__tests__/mermaid.test.d.ts.map +0 -1
- package/dist/render/mermaid/__tests__/mermaid.test.js +0 -74
- package/dist/render/mermaid/__tests__/mermaid.test.js.map +0 -1
- package/dist/storage/__tests__/analysis-tables-migration.test.d.ts +0 -2
- package/dist/storage/__tests__/analysis-tables-migration.test.d.ts.map +0 -1
- package/dist/storage/__tests__/analysis-tables-migration.test.js +0 -44
- package/dist/storage/__tests__/analysis-tables-migration.test.js.map +0 -1
- package/dist/storage/__tests__/graph-store.test.d.ts +0 -2
- package/dist/storage/__tests__/graph-store.test.d.ts.map +0 -1
- package/dist/storage/__tests__/graph-store.test.js +0 -132
- package/dist/storage/__tests__/graph-store.test.js.map +0 -1
- package/dist/storage/__tests__/identifier.test.d.ts +0 -2
- package/dist/storage/__tests__/identifier.test.d.ts.map +0 -1
- package/dist/storage/__tests__/identifier.test.js +0 -35
- package/dist/storage/__tests__/identifier.test.js.map +0 -1
- package/dist/storage/__tests__/load-better-sqlite3.test.d.ts +0 -2
- package/dist/storage/__tests__/load-better-sqlite3.test.d.ts.map +0 -1
- package/dist/storage/__tests__/load-better-sqlite3.test.js +0 -65
- package/dist/storage/__tests__/load-better-sqlite3.test.js.map +0 -1
- package/dist/storage/__tests__/migrations.test.d.ts +0 -2
- package/dist/storage/__tests__/migrations.test.d.ts.map +0 -1
- package/dist/storage/__tests__/migrations.test.js +0 -93
- package/dist/storage/__tests__/migrations.test.js.map +0 -1
- package/dist/storage/__tests__/perf-50k.test.d.ts +0 -2
- package/dist/storage/__tests__/perf-50k.test.d.ts.map +0 -1
- package/dist/storage/__tests__/perf-50k.test.js +0 -67
- package/dist/storage/__tests__/perf-50k.test.js.map +0 -1
- package/dist/storage/__tests__/rename-stability.test.d.ts +0 -2
- package/dist/storage/__tests__/rename-stability.test.d.ts.map +0 -1
- package/dist/storage/__tests__/rename-stability.test.js +0 -135
- package/dist/storage/__tests__/rename-stability.test.js.map +0 -1
- package/dist/storage/__tests__/snapshot-store.test.d.ts +0 -2
- package/dist/storage/__tests__/snapshot-store.test.d.ts.map +0 -1
- package/dist/storage/__tests__/snapshot-store.test.js +0 -117
- package/dist/storage/__tests__/snapshot-store.test.js.map +0 -1
- package/dist/storage/__tests__/snippets.test.d.ts +0 -2
- package/dist/storage/__tests__/snippets.test.d.ts.map +0 -1
- package/dist/storage/__tests__/snippets.test.js +0 -88
- package/dist/storage/__tests__/snippets.test.js.map +0 -1
- package/dist/storage/__tests__/vector-store-load-failure.test.d.ts +0 -2
- package/dist/storage/__tests__/vector-store-load-failure.test.d.ts.map +0 -1
- package/dist/storage/__tests__/vector-store-load-failure.test.js +0 -42
- package/dist/storage/__tests__/vector-store-load-failure.test.js.map +0 -1
- package/dist/storage/__tests__/vector-store.test.d.ts +0 -2
- package/dist/storage/__tests__/vector-store.test.d.ts.map +0 -1
- package/dist/storage/__tests__/vector-store.test.js +0 -87
- package/dist/storage/__tests__/vector-store.test.js.map +0 -1
- package/dist/telemetry/__tests__/event-schema.test.d.ts +0 -2
- package/dist/telemetry/__tests__/event-schema.test.d.ts.map +0 -1
- package/dist/telemetry/__tests__/event-schema.test.js +0 -56
- package/dist/telemetry/__tests__/event-schema.test.js.map +0 -1
- package/dist/telemetry/__tests__/machine-id.test.d.ts +0 -2
- package/dist/telemetry/__tests__/machine-id.test.d.ts.map +0 -1
- package/dist/telemetry/__tests__/machine-id.test.js +0 -33
- package/dist/telemetry/__tests__/machine-id.test.js.map +0 -1
- package/dist/telemetry/__tests__/pipeline.test.d.ts +0 -2
- package/dist/telemetry/__tests__/pipeline.test.d.ts.map +0 -1
- package/dist/telemetry/__tests__/pipeline.test.js +0 -68
- package/dist/telemetry/__tests__/pipeline.test.js.map +0 -1
- package/dist/telemetry/__tests__/sanitizer.test.d.ts +0 -2
- package/dist/telemetry/__tests__/sanitizer.test.d.ts.map +0 -1
- package/dist/telemetry/__tests__/sanitizer.test.js +0 -143
- package/dist/telemetry/__tests__/sanitizer.test.js.map +0 -1
|
@@ -1,20 +1,41 @@
|
|
|
1
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
1
2
|
import { asQualifiedName } from "@ryanstark24/sfgraph-shared";
|
|
2
3
|
import { resolveApexMethodArity } from "../parsers/apex/arity-resolver.js";
|
|
4
|
+
import { retargetApexInterfaceRefs } from "../parsers/apex/interface-ref-resolver.js";
|
|
5
|
+
import { resolveAuraEmbeds } from "../parsers/aura/embed-resolver.js";
|
|
3
6
|
import { resolveCrossFlavor } from "../parsers/cross-flavor-resolver.js";
|
|
4
7
|
import { resolveFlowApexMethods } from "../parsers/flow/invocable-resolver.js";
|
|
5
|
-
import {
|
|
8
|
+
import { resolveFormulaRelationshipHops } from "../parsers/object/formula-hop-resolver.js";
|
|
6
9
|
// Ensure all parsers are registered before we look them up.
|
|
7
10
|
import "../parsers/index.js";
|
|
8
11
|
import { auditDanglingEdges } from "../analyze/audit-graph.js";
|
|
9
12
|
import { populateAnalysisTables } from "../analyze/populate.js";
|
|
10
|
-
import {
|
|
13
|
+
import { REL_TYPES } from "../domain/rel-types.js";
|
|
14
|
+
import { EmbeddingCache, EmbeddingPool, EmbeddingQueue, defaultEmbeddingWorkerUrl, hashEmbedText, } from "../embedding/index.js";
|
|
15
|
+
import { tokenizeIdentifier } from "../embedding/tokenize.js";
|
|
11
16
|
import { resolveOrg } from "../extractors/live-org/auth.js";
|
|
12
17
|
import { bulkRetrieve } from "../extractors/live-org/bulk-retrieve.js";
|
|
13
18
|
import { probeCapabilities } from "../extractors/live-org/capabilities.js";
|
|
14
|
-
import {
|
|
15
|
-
import { startLivenessProbe, } from "../extractors/live-org/liveness.js";
|
|
19
|
+
import { startLivenessProbe } from "../extractors/live-org/liveness.js";
|
|
16
20
|
import { dataPool, metadataPool, toolingPool } from "../extractors/live-org/rate-limit.js";
|
|
21
|
+
import { iterChanges } from "../extractors/live-org/source-member.js";
|
|
22
|
+
// SourceMember MemberType -> the node LABEL the parsers actually store under,
|
|
23
|
+
// for the (rare) types whose label differs from the member type. The LWC parser
|
|
24
|
+
// labels nodes "LWC", not "LightningComponentBundle", so an obsolete-LWC delete
|
|
25
|
+
// keyed by the raw member type was a silent no-op. METADATA_CATEGORY is NOT
|
|
26
|
+
// usable here: its LWC/AURA values ARE the member-type strings, not the labels.
|
|
27
|
+
const MEMBER_TYPE_TO_NODE_LABEL = {
|
|
28
|
+
LightningComponentBundle: "LWC",
|
|
29
|
+
AuraDefinitionBundle: "AuraComponent",
|
|
30
|
+
};
|
|
17
31
|
import { loadAllRules } from "../parsers/rules/_loader.js";
|
|
32
|
+
import { evaluateDeletionGuard, isSkipProtected, resolveMaxDropRatio, skipProtection, } from "./detect-deletions-guard.js";
|
|
33
|
+
import { hasGoodPriorOwned, isParseFailureOnly, isQuarantinedFor, isQuarantinedRow, memberContentHash, memberFailureStamp, memberRowsGeneration, ownedQnames, quarantinedOutcome, } from "./member-rows.js";
|
|
34
|
+
import { OFFLINE_CAPS } from "./offline-org.js";
|
|
35
|
+
import { parseMember } from "./parse-member.js";
|
|
36
|
+
import { parsePool, resolveParseWorkers, } from "./parse-pool.js";
|
|
37
|
+
export { adaptParserInput } from "./parse-member.js";
|
|
38
|
+
import { createIngestProfiler } from "./profile.js";
|
|
18
39
|
const NOOP_LOG = {
|
|
19
40
|
debug: () => { },
|
|
20
41
|
info: () => { },
|
|
@@ -24,23 +45,56 @@ const NOOP_LOG = {
|
|
|
24
45
|
/** Print a consolidated end-of-run summary of metadata types that were
|
|
25
46
|
* skipped during ingest, grouped by failure category, with a targeted
|
|
26
47
|
* remediation block per category. */
|
|
27
|
-
|
|
48
|
+
/**
|
|
49
|
+
* Entries of the previous skip report at `path` (same org) whose label a
|
|
50
|
+
* scoped run did NOT re-run. Missing / malformed / other-org report: none.
|
|
51
|
+
* A `discovery` only-label re-runs every generic:* source (bulkRetrieve), so
|
|
52
|
+
* their entries are not carried either (review N3).
|
|
53
|
+
*/
|
|
54
|
+
export function carriedOverSkips(path, orgId, reRun) {
|
|
55
|
+
try {
|
|
56
|
+
if (!existsSync(path))
|
|
57
|
+
return [];
|
|
58
|
+
const prev = JSON.parse(readFileSync(path, "utf8"));
|
|
59
|
+
if (prev.orgId !== undefined && prev.orgId !== orgId)
|
|
60
|
+
return [];
|
|
61
|
+
return (prev.skips ?? []).filter((s) => s &&
|
|
62
|
+
typeof s.label === "string" &&
|
|
63
|
+
!reRun.has(s.label) &&
|
|
64
|
+
!(reRun.has("discovery") && s.label.startsWith("generic:")));
|
|
65
|
+
}
|
|
66
|
+
catch {
|
|
67
|
+
return [];
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
/** A skip reason without a leading "<label>: " — source errors already name
|
|
71
|
+
* their source, which the summary bullet prints anyway. */
|
|
72
|
+
function bulletReason(s) {
|
|
73
|
+
return s.reason.startsWith(`${s.label}: `) ? s.reason.slice(s.label.length + 2) : s.reason;
|
|
74
|
+
}
|
|
75
|
+
export function printSkipSummary(report, orgAlias) {
|
|
28
76
|
const byCategory = new Map();
|
|
29
77
|
for (const s of report.skips) {
|
|
30
78
|
const arr = byCategory.get(s.category) ?? [];
|
|
31
79
|
arr.push({ label: s.label, reason: s.reason });
|
|
32
80
|
byCategory.set(s.category, arr);
|
|
33
81
|
}
|
|
34
|
-
const
|
|
82
|
+
const unreadableCount = byCategory.get("unreadable")?.length ?? 0;
|
|
83
|
+
const total = report.skips.length - unreadableCount;
|
|
35
84
|
console.log("");
|
|
36
|
-
|
|
37
|
-
|
|
85
|
+
if (total > 0) {
|
|
86
|
+
console.log(`⚠ ${total} metadata ${total === 1 ? "type was" : "types were"} skipped during this ingest.`);
|
|
87
|
+
console.log(" The rest of the graph completed successfully — these types are simply absent.");
|
|
88
|
+
}
|
|
89
|
+
if (unreadableCount > 0) {
|
|
90
|
+
console.log(`⚠ ${unreadableCount} metadata ${unreadableCount === 1 ? "type (or some of its members)" : "types (or some of their members)"} could not be read: the org refused them (unreadable — see below).`);
|
|
91
|
+
}
|
|
38
92
|
if (byCategory.has("insufficient_access")) {
|
|
39
93
|
const list = byCategory.get("insufficient_access") ?? [];
|
|
40
94
|
console.log("");
|
|
41
95
|
console.log(` Insufficient access (${list.length}):`);
|
|
42
96
|
for (const s of list)
|
|
43
|
-
console.log(` • ${s.label}: ${s
|
|
97
|
+
console.log(` • ${s.label}: ${bulletReason(s)}`);
|
|
44
98
|
console.log("");
|
|
45
99
|
console.log(" How to fix permanently:");
|
|
46
100
|
console.log(" 1. Have an admin assign your user a permission set with:");
|
|
@@ -77,112 +131,110 @@ function printSkipSummary(report, orgAlias) {
|
|
|
77
131
|
console.log("");
|
|
78
132
|
console.log(` Network errors (${list.length}):`);
|
|
79
133
|
for (const s of list)
|
|
80
|
-
console.log(` • ${s.label}: ${s
|
|
134
|
+
console.log(` • ${s.label}: ${bulletReason(s)}`);
|
|
81
135
|
console.log(" How to fix: check connectivity to your org and re-run.");
|
|
82
136
|
}
|
|
137
|
+
if (byCategory.has("unreadable")) {
|
|
138
|
+
const list = byCategory.get("unreadable") ?? [];
|
|
139
|
+
console.log("");
|
|
140
|
+
console.log(` Unreadable (org refused; re-run with --retry-skipped to try again) (${list.length}):`);
|
|
141
|
+
for (const s of list)
|
|
142
|
+
console.log(` • ${s.label}: ${bulletReason(s)}`);
|
|
143
|
+
console.log(" The org rejected these reads as invalid, or returned the same error on");
|
|
144
|
+
console.log(" repeated reads, so they were not auto-retried in this run. To try again:");
|
|
145
|
+
console.log(` sfgraph ingest --org ${orgAlias} --retry-skipped`);
|
|
146
|
+
}
|
|
83
147
|
if (byCategory.has("unknown")) {
|
|
84
148
|
const list = byCategory.get("unknown") ?? [];
|
|
85
149
|
console.log("");
|
|
86
150
|
console.log(` Other (${list.length}):`);
|
|
87
151
|
for (const s of list)
|
|
88
|
-
console.log(` • ${s.label}: ${s
|
|
152
|
+
console.log(` • ${s.label}: ${bulletReason(s)}`);
|
|
89
153
|
}
|
|
90
154
|
console.log("");
|
|
91
155
|
}
|
|
92
|
-
/**
|
|
93
|
-
*
|
|
94
|
-
*
|
|
95
|
-
*
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
156
|
+
/**
|
|
157
|
+
* Build the text embedded for a node's semantic vector. Beyond the
|
|
158
|
+
* label/qname/description baseline, this folds in CONFIG VALUES so
|
|
159
|
+
* `find_similar` matches on what a config artifact actually resolves to —
|
|
160
|
+
* a Custom Metadata record's field values, a Custom Label's text, a Custom
|
|
161
|
+
* Setting's rows. Without this, searching "endpoint for product push" can't
|
|
162
|
+
* reach `Endpoint_Url.ProductPush_Acme` because only its name was embedded.
|
|
163
|
+
* Exported for tests.
|
|
164
|
+
*/
|
|
165
|
+
export function buildEmbedText(label, qname, attributes) {
|
|
166
|
+
const a = attributes ?? {};
|
|
167
|
+
const parts = [`${label}: ${qname}`];
|
|
168
|
+
// Word-split the identifier so semantic search matches on word boundaries
|
|
169
|
+
// (`account controller` → AccountController, `customer tier` → Customer_Tier__c).
|
|
170
|
+
// Only the name part after the label prefix; skip if it adds nothing.
|
|
171
|
+
const namePart = qname.includes(":") ? qname.slice(qname.indexOf(":") + 1) : qname;
|
|
172
|
+
const words = tokenizeIdentifier(namePart);
|
|
173
|
+
if (words && words !== namePart.toLowerCase())
|
|
174
|
+
parts.push(words);
|
|
175
|
+
if (typeof a.description === "string" && a.description)
|
|
176
|
+
parts.push(a.description);
|
|
177
|
+
// Custom Metadata record field values: [{field,value}] | {field:value} | object.
|
|
178
|
+
const values = a.values;
|
|
179
|
+
if (Array.isArray(values)) {
|
|
180
|
+
for (const v of values) {
|
|
181
|
+
const f = v.field;
|
|
182
|
+
const val = v.value;
|
|
183
|
+
if (f != null && val != null)
|
|
184
|
+
parts.push(`${String(f)} = ${stringifyConfigVal(val)}`);
|
|
108
185
|
}
|
|
109
186
|
}
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
}
|
|
115
|
-
function adaptParserInput(ref, content) {
|
|
116
|
-
switch (ref.memberType) {
|
|
117
|
-
case "ApexClass": {
|
|
118
|
-
const { body, metaXml } = unwrapApexEnvelope(content);
|
|
119
|
-
return {
|
|
120
|
-
type: "ApexClass",
|
|
121
|
-
input: { className: ref.memberName, body, ...(metaXml ? { metaXml } : {}) },
|
|
122
|
-
};
|
|
187
|
+
else if (values && typeof values === "object") {
|
|
188
|
+
for (const [k, v] of Object.entries(values)) {
|
|
189
|
+
if (v != null && v !== "")
|
|
190
|
+
parts.push(`${k} = ${stringifyConfigVal(v)}`);
|
|
123
191
|
}
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
const
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
catch {
|
|
140
|
-
return null;
|
|
141
|
-
}
|
|
142
|
-
}
|
|
143
|
-
case "Flow":
|
|
144
|
-
return { type: "Flow", input: { fullName: ref.memberName, xml: content } };
|
|
145
|
-
case "CustomObject":
|
|
146
|
-
return { type: "CustomObject", input: { apiName: ref.memberName, objectXml: content } };
|
|
147
|
-
case "Profile":
|
|
148
|
-
return { type: "Profile", input: { name: ref.memberName, xml: content } };
|
|
149
|
-
case "PermissionSet":
|
|
150
|
-
return { type: "PermissionSet", input: { name: ref.memberName, xml: content } };
|
|
151
|
-
case "SharingRules":
|
|
152
|
-
return { type: "SharingRules", input: { object: ref.memberName, xml: content } };
|
|
153
|
-
case "NamedCredential":
|
|
154
|
-
return { type: "NamedCredential", input: { name: ref.memberName, xml: content } };
|
|
155
|
-
case "ExternalServiceRegistration":
|
|
156
|
-
return { type: "ExternalServiceRegistration", input: { name: ref.memberName, xml: content } };
|
|
157
|
-
case "OmniProcess":
|
|
158
|
-
case "OmniUiCard":
|
|
159
|
-
case "OmniDataTransform":
|
|
160
|
-
case "OmniIntegrationProcedure": {
|
|
161
|
-
let metadata = content;
|
|
162
|
-
try {
|
|
163
|
-
metadata = JSON.parse(content);
|
|
164
|
-
}
|
|
165
|
-
catch {
|
|
166
|
-
/* keep raw */
|
|
192
|
+
}
|
|
193
|
+
// Custom Label text value.
|
|
194
|
+
if (typeof a.value === "string" && a.value)
|
|
195
|
+
parts.push(a.value);
|
|
196
|
+
// Custom Setting rows: array of row objects (field=value per row).
|
|
197
|
+
const rows = a.customSettingRows;
|
|
198
|
+
if (Array.isArray(rows)) {
|
|
199
|
+
for (const row of rows) {
|
|
200
|
+
if (row && typeof row === "object") {
|
|
201
|
+
const kv = Object.entries(row)
|
|
202
|
+
.filter(([, v]) => v != null && v !== "")
|
|
203
|
+
.map(([k, v]) => `${k}=${stringifyConfigVal(v)}`)
|
|
204
|
+
.join(", ");
|
|
205
|
+
if (kv)
|
|
206
|
+
parts.push(kv);
|
|
167
207
|
}
|
|
168
|
-
return { type: ref.memberType, input: { name: ref.memberName, metadata } };
|
|
169
208
|
}
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
209
|
+
}
|
|
210
|
+
// Structural attributes that make automation findable by WHAT it acts on:
|
|
211
|
+
// a trigger's object + DML events ("on Account before insert"), a Flow's
|
|
212
|
+
// process type ("AutoLaunchedFlow"/"Flow"). Gated by label — CustomField (and
|
|
213
|
+
// other nodes) also carry an `object` attribute (their parent SObject), and
|
|
214
|
+
// appending "on Account" to every field was pure noise in the embedding.
|
|
215
|
+
if (label === "ApexTrigger" && typeof a.object === "string" && a.object) {
|
|
216
|
+
const ev = Array.isArray(a.events)
|
|
217
|
+
? a.events.filter((x) => typeof x === "string").join(" ")
|
|
218
|
+
: "";
|
|
219
|
+
parts.push(`on ${a.object}${ev ? ` ${ev}` : ""}`);
|
|
220
|
+
}
|
|
221
|
+
if (label === "Flow" && typeof a.processType === "string" && a.processType) {
|
|
222
|
+
parts.push(a.processType);
|
|
223
|
+
}
|
|
224
|
+
return parts.join("\n");
|
|
225
|
+
}
|
|
226
|
+
function stringifyConfigVal(v) {
|
|
227
|
+
if (v == null)
|
|
228
|
+
return "";
|
|
229
|
+
if (typeof v === "object") {
|
|
230
|
+
try {
|
|
231
|
+
return JSON.stringify(v);
|
|
232
|
+
}
|
|
233
|
+
catch {
|
|
234
|
+
return String(v);
|
|
182
235
|
}
|
|
183
|
-
default:
|
|
184
|
-
return null;
|
|
185
236
|
}
|
|
237
|
+
return String(v);
|
|
186
238
|
}
|
|
187
239
|
export async function liveIngest(opts) {
|
|
188
240
|
const logger = opts.logger ?? NOOP_LOG;
|
|
@@ -206,617 +258,1557 @@ export async function liveIngest(opts) {
|
|
|
206
258
|
apiVersion,
|
|
207
259
|
createdAt: now,
|
|
208
260
|
});
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
//
|
|
212
|
-
//
|
|
261
|
+
// W1.5-08: flag the org row as having an in-flight ingest before any
|
|
262
|
+
// extractor work. Done after upsertOrg so the row definitely exists.
|
|
263
|
+
// The matching markSyncComplete / markSyncFailed calls below are wired
|
|
264
|
+
// into a try/catch around the whole ingest body so concurrent MCP
|
|
265
|
+
// readers always see in_progress flip back to 0 even when ingest throws.
|
|
266
|
+
// Failure to record sync-start is non-fatal — log and continue rather
|
|
267
|
+
// than aborting an ingest because the bookkeeping column is unhappy.
|
|
268
|
+
const ingestStartedIso = new Date(now).toISOString();
|
|
213
269
|
try {
|
|
214
|
-
|
|
215
|
-
const discovered = await discoverMetadataTypes(resolved.conn, apiVersion);
|
|
216
|
-
logger.info("live-ingest: discovered metadata types", { count: discovered.length });
|
|
270
|
+
graph.markSyncStarted(resolved.orgId, ingestStartedIso);
|
|
217
271
|
}
|
|
218
272
|
catch (e) {
|
|
219
|
-
logger.warn("live-ingest:
|
|
273
|
+
logger.warn("live-ingest: markSyncStarted failed", { err: e.message });
|
|
220
274
|
}
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
275
|
+
// Opt-in profiling (SFGRAPH_INGEST_PROFILE=<file>); no-op otherwise.
|
|
276
|
+
// Created before the try so the finally can flush on failure too.
|
|
277
|
+
const profiler = createIngestProfiler();
|
|
278
|
+
// Embedding resources, owned here so the finally can release them on any path.
|
|
279
|
+
let embedQueue = null;
|
|
280
|
+
let ownedEmbedder = null;
|
|
281
|
+
let embedCache = null;
|
|
282
|
+
// W1.5-08: the rest of the ingest runs inside this try so a thrown
|
|
283
|
+
// exception still flips `sync_in_progress` back to 0 via markSyncFailed.
|
|
284
|
+
// Per-resolver try/catch isolation inside bulkRetrieve / live-ingest is
|
|
285
|
+
// preserved — this wrapper only catches exceptions that escape *all* of
|
|
286
|
+
// those inner handlers (i.e. genuinely fatal failures).
|
|
287
|
+
try {
|
|
288
|
+
const caps = opts.metadataSource ? OFFLINE_CAPS : await probeCapabilities(resolved.conn);
|
|
289
|
+
logger.info("live-ingest: probed capabilities", { caps });
|
|
290
|
+
// Discovery is also invoked implicitly inside bulkRetrieve, but logging the
|
|
291
|
+
// type-count up-front gives operators a quick coverage signal.
|
|
292
|
+
if (!opts.metadataSource) {
|
|
293
|
+
try {
|
|
294
|
+
const { discoverMetadataTypes } = await import("../extractors/live-org/discovery.js");
|
|
295
|
+
const discovered = await discoverMetadataTypes(resolved.conn, apiVersion);
|
|
296
|
+
logger.info("live-ingest: discovered metadata types", { count: discovered.length });
|
|
297
|
+
}
|
|
298
|
+
catch (e) {
|
|
299
|
+
logger.warn("live-ingest: metadata.describe failed", { err: e.message });
|
|
300
|
+
}
|
|
233
301
|
}
|
|
234
|
-
|
|
235
|
-
|
|
302
|
+
const existing = graph.getOrg(resolved.orgId);
|
|
303
|
+
const requestedMode = opts.mode ?? "auto";
|
|
304
|
+
let mode;
|
|
305
|
+
if (opts.metadataSource) {
|
|
306
|
+
mode = "full";
|
|
236
307
|
}
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
308
|
+
else if (requestedMode === "auto") {
|
|
309
|
+
mode = existing?.lastSyncedAt && caps.sourceTracking ? "incremental" : "full";
|
|
310
|
+
}
|
|
311
|
+
else {
|
|
312
|
+
mode = requestedMode;
|
|
313
|
+
}
|
|
314
|
+
if (!opts.skipSnapshot && opts.snapshotStore) {
|
|
315
|
+
try {
|
|
316
|
+
opts.snapshotStore.createSnapshot(resolved.orgId, `pre-sync-${new Date(now).toISOString()}`, true);
|
|
317
|
+
}
|
|
318
|
+
catch (e) {
|
|
319
|
+
logger.warn("live-ingest: snapshot failed", { err: e.message });
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
let membersProcessed = 0;
|
|
323
|
+
let parseErrors = 0;
|
|
324
|
+
let deletions = 0;
|
|
325
|
+
// Incremental detects changed/added members but cannot fetch their content
|
|
326
|
+
// yet, so it skips them. If we then advanced the watermark past them, the
|
|
327
|
+
// next incremental would start AFTER those changes and never see them —
|
|
328
|
+
// silent permanent data loss. Track the count so we can HOLD the watermark.
|
|
329
|
+
let incrementalUnfetched = 0;
|
|
330
|
+
// Set in the full-sync branch if the bulkRetrieve stream itself throws.
|
|
331
|
+
// Detect-deletions reads it post-fan-out to bail out instead of mass-
|
|
332
|
+
// wiping nodes that were never visited due to the abort.
|
|
333
|
+
let streamAborted = false;
|
|
334
|
+
const touchedQnames = new Set();
|
|
335
|
+
// W1.5-07: maintain per-label touched-qnames sets in parallel with the
|
|
336
|
+
// flat `touchedQnames` set so the deletion-sweep guard can compute
|
|
337
|
+
// priorCount/touchedCount per label without re-deriving labels later.
|
|
338
|
+
const touchedQnamesByLabel = new Map();
|
|
339
|
+
// Skip/warning report collected across all branches. Populated by the
|
|
340
|
+
// full-sync fan-out via bulkRetrieve's onError callback; incremental mode
|
|
341
|
+
// currently emits no entries (single-stream iterChanges has no per-source
|
|
342
|
+
// failure surface). Returned on LiveIngestResult.warnings so MCP consumers
|
|
343
|
+
// can read what was skipped without parsing stdout.
|
|
344
|
+
const skipReport = { skips: [] };
|
|
345
|
+
// Phase 1.5 W1.5-02: namespaced wedge warnings populated by bulkRetrieve's
|
|
346
|
+
// soft-isolate path (`wedge:<source>:<stage>:<detail>`). Concatenated with
|
|
347
|
+
// the legacy `${label}: ${reason}` strings from skipReport.skips into the
|
|
348
|
+
// final LiveIngestResult.warnings array. See bulk-retrieve.ts WedgeCoordinator.
|
|
349
|
+
const wedgeWarnings = [];
|
|
350
|
+
// Task 10 rebuild fast path. On a full ingest into an org whose graph is
|
|
351
|
+
// empty at start, run the fan-out inside the store's bulk-load window:
|
|
352
|
+
// deferred secondary indexes, synchronous=OFF, one transaction per 2,000
|
|
353
|
+
// members, and no per-member delete probes for keys not yet written (the
|
|
354
|
+
// store tracks what this run wrote, so repeats are still reconciled and
|
|
355
|
+
// the graph is identical to the slow path). SFGRAPH_DISABLE_FASTPATH=1
|
|
356
|
+
// forces the per-member path (used by the same-output test and gate).
|
|
357
|
+
const fastPathDisabled = process.env.SFGRAPH_DISABLE_FASTPATH === "1";
|
|
358
|
+
const graphEmptyAtStart = !fastPathDisabled &&
|
|
359
|
+
mode === "full" &&
|
|
360
|
+
opts.rebuild !== false &&
|
|
361
|
+
typeof graph.beginBulk === "function" &&
|
|
362
|
+
typeof graph.endBulk === "function" &&
|
|
363
|
+
typeof graph.isOrgEmpty === "function" &&
|
|
364
|
+
graph.isOrgEmpty(resolved.orgId);
|
|
365
|
+
// A vector store on its own connection would hit SQLITE_BUSY behind the
|
|
366
|
+
// held bulk transaction, so keep per-member commits in that case.
|
|
367
|
+
const vectorDb = opts.vectorStore?.db;
|
|
368
|
+
const graphDb = graph.db;
|
|
369
|
+
const sharedConnection = vectorDb === undefined || vectorDb === graphDb;
|
|
370
|
+
// Every other full ingest (a normal sync, a delta's merge) writes into a
|
|
371
|
+
// graph that MCP servers may be reading, so the bulk window's relaxed
|
|
372
|
+
// pragmas and dropped indexes are not an option. It still groups members
|
|
373
|
+
// into plain transactions (store.beginBatch: BEGIN IMMEDIATE, committed
|
|
374
|
+
// only between members, at most ~200 ms each), which removes most of the
|
|
375
|
+
// per-member commit cost. Same SQLITE_BUSY rule as the bulk window: a
|
|
376
|
+
// vector store on its own connection keeps per-member commits.
|
|
377
|
+
const batchWrites = !fastPathDisabled &&
|
|
378
|
+
!graphEmptyAtStart &&
|
|
379
|
+
mode === "full" &&
|
|
380
|
+
sharedConnection &&
|
|
381
|
+
typeof graph.beginBatch === "function" &&
|
|
382
|
+
typeof graph.endBatch === "function";
|
|
383
|
+
let bulkOpen = false;
|
|
384
|
+
let batchOpen = false;
|
|
385
|
+
let bulkFailure = null;
|
|
386
|
+
const openBulk = () => {
|
|
387
|
+
if (graphEmptyAtStart && !bulkOpen) {
|
|
388
|
+
graph.beginBulk?.({
|
|
389
|
+
freshOrgId: resolved.orgId,
|
|
390
|
+
...(sharedConnection ? {} : { batchTransactions: false }),
|
|
391
|
+
});
|
|
392
|
+
bulkOpen = true;
|
|
393
|
+
}
|
|
394
|
+
else if (batchWrites && !batchOpen) {
|
|
395
|
+
try {
|
|
396
|
+
graph.beginBatch?.();
|
|
397
|
+
batchOpen = true;
|
|
398
|
+
}
|
|
399
|
+
catch (e) {
|
|
400
|
+
// e.g. the caller holds a transaction: keep per-member commits.
|
|
401
|
+
logger.debug("live-ingest: write batching unavailable", { err: e.message });
|
|
276
402
|
}
|
|
403
|
+
}
|
|
404
|
+
};
|
|
405
|
+
const closeBulk = () => {
|
|
406
|
+
if (batchOpen) {
|
|
407
|
+
batchOpen = false;
|
|
408
|
+
graph.endBatch?.();
|
|
409
|
+
}
|
|
410
|
+
if (!bulkOpen)
|
|
411
|
+
return;
|
|
412
|
+
bulkOpen = false;
|
|
413
|
+
graph.endBulk?.();
|
|
414
|
+
};
|
|
415
|
+
const parseCtxBase = {
|
|
416
|
+
orgId: resolved.orgId,
|
|
417
|
+
parseTimestamp: new Date(now).toISOString(),
|
|
418
|
+
namespace: null,
|
|
419
|
+
logger,
|
|
420
|
+
};
|
|
421
|
+
// Task 11a: embeddings run off the graph-ready path. Inference happens in
|
|
422
|
+
// a worker thread (compiled builds) or an injected embedder, with up to 4
|
|
423
|
+
// batches in flight to keep it fed. The queue is HELD until graph ready:
|
|
424
|
+
// texts buffer (bounded by backpressure, which still forces batches out)
|
|
425
|
+
// and no inference competes for CPU with the graph build — measured, an
|
|
426
|
+
// overlapping worker slowed fan-out by ~45%. A persistent text → vector
|
|
427
|
+
// cache makes --rebuild skip the model for unchanged texts. The org is flagged `embeddings_pending` until every
|
|
428
|
+
// vector is written, so find_similar can fall back to text search and the
|
|
429
|
+
// next ingest can fill whatever a no-wait / failed run left behind. The
|
|
430
|
+
// flag is set lazily, when the queue first has real vector work (a text
|
|
431
|
+
// past the unchanged check at dispatch, or live work abandoned), so a
|
|
432
|
+
// re-sync with nothing to embed keeps find_similar's vector leg on.
|
|
433
|
+
let embedError = null;
|
|
434
|
+
let wasEmbeddingsPending = false;
|
|
435
|
+
if (opts.vectorStore) {
|
|
436
|
+
const workerUrl = opts.embedder ? null : defaultEmbeddingWorkerUrl();
|
|
437
|
+
ownedEmbedder = opts.embedder ? null : workerUrl ? new EmbeddingPool({ workerUrl }) : null;
|
|
438
|
+
const embedder = opts.embedder ?? ownedEmbedder;
|
|
439
|
+
if (opts.embeddingCacheFile) {
|
|
440
|
+
try {
|
|
441
|
+
embedCache = EmbeddingCache.open(opts.embeddingCacheFile);
|
|
442
|
+
}
|
|
443
|
+
catch (e) {
|
|
444
|
+
logger.warn("live-ingest: embedding cache unavailable", { err: e.message });
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
wasEmbeddingsPending = graph.getEmbeddingStatus?.(resolved.orgId)?.pending ?? false;
|
|
448
|
+
const maxPendingChars = Number.parseInt(process.env.SFGRAPH_EMBED_MAX_PENDING_CHARS ?? "", 10);
|
|
449
|
+
embedQueue = new EmbeddingQueue({
|
|
450
|
+
vectorStore: opts.vectorStore,
|
|
451
|
+
hold: true,
|
|
452
|
+
...(maxPendingChars > 0 ? { maxPendingChars } : {}),
|
|
453
|
+
onWork: () => {
|
|
454
|
+
if (!wasEmbeddingsPending)
|
|
455
|
+
graph.setEmbeddingsPending?.(resolved.orgId, true);
|
|
456
|
+
},
|
|
457
|
+
onCacheError: (err) => {
|
|
458
|
+
logger.warn("live-ingest: embedding cache write failed", { err: err.message });
|
|
459
|
+
},
|
|
460
|
+
...(embedder ? { embed: (t) => embedder.embed(t), maxInFlight: 4 } : {}),
|
|
461
|
+
...(embedCache ? { cache: embedCache } : {}),
|
|
462
|
+
onError: (err) => {
|
|
463
|
+
if (!embedError)
|
|
464
|
+
embedError = err;
|
|
465
|
+
logger.warn("live-ingest: embedding batch failed", { err: err.message });
|
|
466
|
+
},
|
|
277
467
|
});
|
|
278
468
|
}
|
|
279
|
-
|
|
469
|
+
const handleParsed = (parsed) => {
|
|
470
|
+
if (parsed.nodes.length) {
|
|
471
|
+
graph.mergeNodes(parsed.nodes);
|
|
472
|
+
for (const n of parsed.nodes) {
|
|
473
|
+
const qn = String(n.qualifiedName);
|
|
474
|
+
touchedQnames.add(qn);
|
|
475
|
+
// W1.5-07: also bucket by label for the per-label deletion guard.
|
|
476
|
+
let bucket = touchedQnamesByLabel.get(n.label);
|
|
477
|
+
if (!bucket) {
|
|
478
|
+
bucket = new Set();
|
|
479
|
+
touchedQnamesByLabel.set(n.label, bucket);
|
|
480
|
+
}
|
|
481
|
+
bucket.add(qn);
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
// A parser's ParseResult is the COMPLETE authoritative outgoing-edge set
|
|
485
|
+
// for the member it parsed, so reconcile: prune any prior outgoing edge
|
|
486
|
+
// from these sources that the member no longer emits (fixes "immortal
|
|
487
|
+
// edges" — a SOQL/DML/CALLS that lingered after the code dropped it on a
|
|
488
|
+
// non-rebuild re-sync). Resolver post-passes (arity / overlap / reflection)
|
|
489
|
+
// re-add their cross-cutting edges afterward, so this is safe.
|
|
490
|
+
// N-5: a method the member still emits but that now has NO outgoing
|
|
491
|
+
// edge (its only call was removed) is not a source in `parsed.edges`,
|
|
492
|
+
// so it is named explicitly and its stale edges are pruned too.
|
|
493
|
+
const methodSrcs = [];
|
|
280
494
|
for (const n of parsed.nodes) {
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
495
|
+
if (n.label === "ApexMethod" || n.label === "TestMethod") {
|
|
496
|
+
methodSrcs.push(String(n.qualifiedName));
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
if (parsed.edges.length || methodSrcs.length) {
|
|
500
|
+
graph.mergeEdges(parsed.edges, {
|
|
501
|
+
reconcileSources: true,
|
|
502
|
+
...(methodSrcs.length
|
|
503
|
+
? { reconcileExtraSrcs: { orgId: resolved.orgId, qnames: methodSrcs } }
|
|
504
|
+
: {}),
|
|
288
505
|
});
|
|
289
506
|
}
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
507
|
+
if (parsed.snippets?.length) {
|
|
508
|
+
graph.transaction(() => {
|
|
509
|
+
for (const s of parsed.snippets ?? []) {
|
|
510
|
+
graph.upsertSnippet(s);
|
|
511
|
+
}
|
|
512
|
+
});
|
|
513
|
+
}
|
|
514
|
+
// Fold the stored source snippet into the search text for code nodes so
|
|
515
|
+
// search matches on what the code DOES, not just its name/attrs. (The
|
|
516
|
+
// snippet text lives in parsed.snippets, not on the node.) Bounded to a
|
|
517
|
+
// few KB — MiniLM truncates to its token window anyway, and we keep the
|
|
518
|
+
// structured buildEmbedText prefix first so identity stays dominant. The
|
|
519
|
+
// SAME text feeds both the vector index and the FTS keyword index; FTS is
|
|
520
|
+
// populated regardless of embedQueue so it works as the embedder-down
|
|
521
|
+
// fallback leg of hybrid search.
|
|
522
|
+
if (parsed.nodes.length) {
|
|
523
|
+
const snippetByQname = new Map();
|
|
524
|
+
for (const s of parsed.snippets ?? []) {
|
|
525
|
+
snippetByQname.set(String(s.qualifiedName), s.sourceText);
|
|
526
|
+
}
|
|
527
|
+
for (const n of parsed.nodes) {
|
|
528
|
+
const qn = String(n.qualifiedName);
|
|
529
|
+
let text = buildEmbedText(n.label, qn, n.attributes);
|
|
530
|
+
const snippet = snippetByQname.get(qn);
|
|
531
|
+
if (snippet)
|
|
532
|
+
text += `\n${snippet.slice(0, 4000)}`;
|
|
533
|
+
graph.upsertNodeFts(n.orgId, n.qualifiedName, n.label, text);
|
|
534
|
+
if (embedQueue) {
|
|
535
|
+
embedQueue.push({ qname: qn, text, orgId: String(n.orgId), label: n.label });
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
};
|
|
540
|
+
// Purge a deleted node's vector so it stops surfacing as a phantom
|
|
541
|
+
// find_similar hit. The vec0 table lives in the VectorStore (not the graph
|
|
542
|
+
// store); feature-detect since lightweight sinks may not implement delete.
|
|
543
|
+
const purgeNodeVector = (qname) => {
|
|
544
|
+
const sink = opts.vectorStore;
|
|
545
|
+
if (sink && typeof sink.deleteNodeVector === "function") {
|
|
546
|
+
try {
|
|
547
|
+
sink.deleteNodeVector(resolved.orgId, qname);
|
|
548
|
+
}
|
|
549
|
+
catch (e) {
|
|
550
|
+
logger.warn("live-ingest: vector purge failed", {
|
|
551
|
+
qname: String(qname),
|
|
552
|
+
err: e.message,
|
|
553
|
+
});
|
|
554
|
+
}
|
|
555
|
+
}
|
|
556
|
+
};
|
|
557
|
+
const debugProcess = process.env.SFGRAPH_DEBUG_INGEST === "1";
|
|
558
|
+
// Per-member timings, filled by applyMember (parse vs. store write) and
|
|
559
|
+
// consumed by the member loop. Reset on every call. These are shared
|
|
560
|
+
// closure state: applyMember must NOT be called concurrently.
|
|
561
|
+
let lastParseMs = 0;
|
|
562
|
+
let lastWriteMs = 0;
|
|
563
|
+
// Apply one member's parse outcome to the graph (or delete an obsolete
|
|
564
|
+
// member). Always on the main thread, in member order: the parse may have
|
|
565
|
+
// run in a worker (parsePool), the writes never do.
|
|
566
|
+
const applyMember = (ref, outcome) => {
|
|
567
|
+
const qnameForLog = `${ref.memberType}:${ref.memberName}`;
|
|
568
|
+
lastParseMs = 0;
|
|
569
|
+
lastWriteMs = 0;
|
|
570
|
+
if (ref.obsolete) {
|
|
571
|
+
// Build the qname with the node's LABEL, not the raw SourceMember type:
|
|
572
|
+
// e.g. LightningComponentBundle is stored as `LWC:<name>`, so the old
|
|
573
|
+
// `${memberType}:${name}` never matched and LWC deletes were silent
|
|
574
|
+
// no-ops. Fall back to memberType when no mapping exists.
|
|
575
|
+
const label = MEMBER_TYPE_TO_NODE_LABEL[ref.memberType] ?? ref.memberType;
|
|
576
|
+
const qname = asQualifiedName(`${label}:${ref.memberName}`);
|
|
577
|
+
if (debugProcess)
|
|
578
|
+
console.log(`ingest: [trace] delete ← ${qnameForLog}`);
|
|
579
|
+
// Cascade to contained children (a class/trigger owns ApexMethod /
|
|
580
|
+
// TestMethod nodes via CONTAINS_METHOD). Deleting only the parent left
|
|
581
|
+
// the method nodes — and their edges/snippets/vectors — orphaned.
|
|
582
|
+
for (const e of graph.listEdgesFrom(resolved.orgId, qname, REL_TYPES.CONTAINS_METHOD)) {
|
|
583
|
+
const child = e.dstQualifiedName;
|
|
584
|
+
graph.deleteEdgesFor(resolved.orgId, child);
|
|
585
|
+
graph.deleteNode(resolved.orgId, child);
|
|
586
|
+
purgeNodeVector(child);
|
|
587
|
+
}
|
|
588
|
+
graph.deleteEdgesFor(resolved.orgId, qname);
|
|
589
|
+
graph.deleteNode(resolved.orgId, qname);
|
|
590
|
+
purgeNodeVector(qname);
|
|
591
|
+
if (debugProcess)
|
|
592
|
+
console.log(`ingest: [trace] delete ✓ ${qnameForLog}`);
|
|
593
|
+
deletions += 1;
|
|
594
|
+
return;
|
|
595
|
+
}
|
|
596
|
+
// Building the parser input threw: surfaces to the member loop's catch.
|
|
597
|
+
if (outcome.kind === "error" && outcome.phase === "adapt")
|
|
598
|
+
throw new Error(outcome.message);
|
|
599
|
+
if (outcome.kind === "skip")
|
|
600
|
+
return;
|
|
601
|
+
// Emitted when the result is applied (in member order); the parse itself
|
|
602
|
+
// may have run earlier in a worker.
|
|
319
603
|
if (debugProcess)
|
|
320
604
|
console.log(`ingest: [trace] parse ← ${qnameForLog}`);
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
605
|
+
lastParseMs = outcome.parseMs;
|
|
606
|
+
try {
|
|
607
|
+
if (outcome.kind === "error") {
|
|
608
|
+
const err = new Error(outcome.message);
|
|
609
|
+
if (outcome.stack)
|
|
610
|
+
err.stack = outcome.stack;
|
|
611
|
+
throw err;
|
|
612
|
+
}
|
|
613
|
+
const result = outcome.result;
|
|
614
|
+
if (debugProcess)
|
|
615
|
+
console.log(`ingest: [trace] parse ✓ ${qnameForLog} (nodes=${result.nodes.length} edges=${result.edges.length})`);
|
|
616
|
+
if (debugProcess)
|
|
617
|
+
console.log(`ingest: [trace] graph-merge ← ${qnameForLog}`);
|
|
618
|
+
const tWrite = performance.now();
|
|
619
|
+
try {
|
|
620
|
+
handleParsed(result);
|
|
621
|
+
}
|
|
622
|
+
finally {
|
|
623
|
+
lastWriteMs = performance.now() - tWrite;
|
|
624
|
+
}
|
|
625
|
+
if (debugProcess)
|
|
626
|
+
console.log(`ingest: [trace] graph-merge ✓ ${qnameForLog}`);
|
|
627
|
+
membersProcessed += 1;
|
|
340
628
|
}
|
|
629
|
+
catch (e) {
|
|
630
|
+
parseErrors += 1;
|
|
631
|
+
logger.warn("live-ingest: parser failure", {
|
|
632
|
+
type: ref.memberType,
|
|
633
|
+
name: ref.memberName,
|
|
634
|
+
err: e.message,
|
|
635
|
+
});
|
|
636
|
+
if (debugProcess) {
|
|
637
|
+
console.error(`ingest: [trace] FAILURE ${qnameForLog}: ${e.message}\n${e.stack}`);
|
|
638
|
+
}
|
|
639
|
+
}
|
|
640
|
+
};
|
|
641
|
+
// Task 11c: member rows (delta ingest's record of what each member owns)
|
|
642
|
+
// and the ghost sweep, on a full ingest of a metadataSource. The rows are
|
|
643
|
+
// cleared now and rewritten only once the ingest completes, so an
|
|
644
|
+
// interrupted run leaves no rows and the next delta falls back to a full
|
|
645
|
+
// rebuild. The ghost sweep deletes what a member owned last run that no
|
|
646
|
+
// member owns now (a deleted / renamed file), before the resolvers run.
|
|
647
|
+
if (opts.delta && (!opts.metadataSource || mode !== "full")) {
|
|
648
|
+
throw new Error("liveIngest: delta hooks need a metadataSource and mode 'full'");
|
|
341
649
|
}
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
//
|
|
356
|
-
//
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
const
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
.
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
...
|
|
379
|
-
sourceUri: "mcd-baseline://tooling-soql",
|
|
380
|
-
};
|
|
381
|
-
const mcd = await runMcdBaseline(resolved.conn, {
|
|
382
|
-
orgId: resolved.orgId,
|
|
383
|
-
ctx: mcdCtx,
|
|
384
|
-
onError: (label, err) => {
|
|
385
|
-
skipReport.skips.push({
|
|
386
|
-
label,
|
|
387
|
-
reason: err.message,
|
|
388
|
-
category: "unknown",
|
|
389
|
-
});
|
|
390
|
-
},
|
|
650
|
+
const deltaHooks = opts.delta;
|
|
651
|
+
const priorRows = !deltaHooks &&
|
|
652
|
+
opts.metadataSource &&
|
|
653
|
+
mode === "full" &&
|
|
654
|
+
graph.listMemberRows &&
|
|
655
|
+
graph.replaceMemberRows
|
|
656
|
+
? graph.listMemberRows(resolved.orgId)
|
|
657
|
+
: null;
|
|
658
|
+
if (priorRows && priorRows.length > 0)
|
|
659
|
+
graph.clearMemberRows?.(resolved.orgId);
|
|
660
|
+
const memberRows = priorRows ? new Map() : null;
|
|
661
|
+
const priorRowByKey = new Map((priorRows ?? []).map((r) => [r.key, r]));
|
|
662
|
+
const keptPrior = [];
|
|
663
|
+
// T15 N5-1: members quarantined by earlier runs (workers kept failing on
|
|
664
|
+
// that exact content) are not parsed again until their content changes.
|
|
665
|
+
const quarantinedKeys = new Set((priorRows ?? []).filter((r) => isQuarantinedRow(r)).map((r) => r.key));
|
|
666
|
+
const quarantinePreset = (m) => quarantinedKeys.has(m.ref.sourceUri) &&
|
|
667
|
+
isQuarantinedFor(priorRowByKey.get(m.ref.sourceUri), memberContentHash(m.content))
|
|
668
|
+
? quarantinedOutcome()
|
|
669
|
+
: undefined;
|
|
670
|
+
const recordMemberRow = (member, outcome, errored) => {
|
|
671
|
+
if (!memberRows)
|
|
672
|
+
return;
|
|
673
|
+
const key = member.ref.sourceUri;
|
|
674
|
+
const prior = priorRowByKey.get(key);
|
|
675
|
+
const owned = outcome.kind === "parsed" ? ownedQnames(outcome.result) : [];
|
|
676
|
+
if (errored || (isParseFailureOnly(owned) && hasGoodPriorOwned(prior))) {
|
|
677
|
+
// Keep what it owned (its nodes are not re-emitted, nor swept). A
|
|
678
|
+
// member that parsed to ParseError only (N2-1) or whose parse threw
|
|
679
|
+
// records the hash of its broken content and is skipped until it
|
|
680
|
+
// changes; one whose merge failed is retried by every delta.
|
|
681
|
+
if (hasGoodPriorOwned(prior))
|
|
682
|
+
keptPrior.push(key);
|
|
683
|
+
memberRows.set(key, {
|
|
684
|
+
key,
|
|
685
|
+
...memberFailureStamp(member, outcome, errored, prior),
|
|
686
|
+
owned: [...new Set([...(prior?.owned ?? []), ...owned])],
|
|
391
687
|
});
|
|
392
|
-
|
|
393
|
-
|
|
688
|
+
return;
|
|
689
|
+
}
|
|
690
|
+
memberRows.set(key, {
|
|
691
|
+
key,
|
|
692
|
+
fingerprint: member.fingerprint ?? "",
|
|
693
|
+
hash: memberContentHash(member.content),
|
|
694
|
+
owned,
|
|
695
|
+
});
|
|
696
|
+
};
|
|
697
|
+
const finishMemberRows = () => {
|
|
698
|
+
if (!memberRows || !priorRows)
|
|
699
|
+
return;
|
|
700
|
+
if (streamAborted) {
|
|
701
|
+
memberRows.clear();
|
|
702
|
+
return;
|
|
703
|
+
}
|
|
704
|
+
if (priorRows.length === 0)
|
|
705
|
+
return;
|
|
706
|
+
// Members the source indexed but could not load (unreadable, malformed)
|
|
707
|
+
// keep their previous graph and row (retried by the next delta).
|
|
708
|
+
const unloaded = opts.metadataSource?.unloadedKeys?.();
|
|
709
|
+
if (unloaded) {
|
|
710
|
+
for (const r of priorRows) {
|
|
711
|
+
if (!unloaded.has(r.key) || memberRows.has(r.key))
|
|
712
|
+
continue;
|
|
713
|
+
memberRows.set(r.key, { ...r, fingerprint: "" });
|
|
714
|
+
keptPrior.push(r.key);
|
|
394
715
|
}
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
716
|
+
}
|
|
717
|
+
if (keptPrior.length > 0) {
|
|
718
|
+
wedgeWarnings.push(`wedge:ingest:kept-prior:members=${keptPrior.length}`);
|
|
719
|
+
logger.warn("live-ingest: unreadable / unparseable members keep their previous graph", {
|
|
720
|
+
members: keptPrior.slice(0, 20),
|
|
398
721
|
});
|
|
399
722
|
}
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
//
|
|
455
|
-
//
|
|
456
|
-
//
|
|
457
|
-
//
|
|
458
|
-
//
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
if (idleSec < 10)
|
|
464
|
-
return; // suppress chatter when records are flowing
|
|
465
|
-
if (debug) {
|
|
466
|
-
const mem = process.memoryUsage();
|
|
467
|
-
const heapMb = Math.round(mem.heapUsed / 1024 / 1024);
|
|
468
|
-
const rssMb = Math.round(mem.rss / 1024 / 1024);
|
|
469
|
-
const externalMb = Math.round(mem.external / 1024 / 1024);
|
|
470
|
-
console.log(`ingest: [heartbeat] processed=${progressCount} lastSource=${lastMemberLabel} idle=${idleSec}s heap=${heapMb}MB rss=${rssMb}MB ext=${externalMb}MB`);
|
|
723
|
+
if (memberRows.size === 0) {
|
|
724
|
+
// Same guard as detect-deletions' empty-stream refusal: a source that
|
|
725
|
+
// yielded no members at all (wrong path, unreadable tree) must not
|
|
726
|
+
// wipe the graph. No rows are written, so the next delta rebuilds.
|
|
727
|
+
wedgeWarnings.push(`wedge:ghost-sweep:refuse:reason=empty-stream:priorMembers=${priorRows.length}`);
|
|
728
|
+
logger.warn("live-ingest: ghost sweep refused (the source yielded no members)", {
|
|
729
|
+
priorMembers: priorRows.length,
|
|
730
|
+
});
|
|
731
|
+
return;
|
|
732
|
+
}
|
|
733
|
+
// Qnames match case-insensitively (the store's NOCASE): `ApexClass:Util`
|
|
734
|
+
// re-emitted as `ApexClass:UTIL` is the same node, not a ghost.
|
|
735
|
+
const ownedNow = new Set();
|
|
736
|
+
for (const r of memberRows.values())
|
|
737
|
+
for (const q of r.owned)
|
|
738
|
+
ownedNow.add(q.toLowerCase());
|
|
739
|
+
const ghosts = [];
|
|
740
|
+
for (const r of priorRows) {
|
|
741
|
+
for (const q of r.owned)
|
|
742
|
+
if (!ownedNow.has(q.toLowerCase()))
|
|
743
|
+
ghosts.push(q);
|
|
744
|
+
}
|
|
745
|
+
if (ghosts.length === 0)
|
|
746
|
+
return;
|
|
747
|
+
let removed = 0;
|
|
748
|
+
graph.transaction(() => {
|
|
749
|
+
graph.deleteEdgesFrom?.(resolved.orgId, ghosts);
|
|
750
|
+
graph.deleteReflectionEdges?.(resolved.orgId, { dsts: ghosts });
|
|
751
|
+
for (const q of new Set(ghosts)) {
|
|
752
|
+
const qn = asQualifiedName(q);
|
|
753
|
+
if (!graph.getNode(resolved.orgId, qn))
|
|
754
|
+
continue;
|
|
755
|
+
graph.deleteNode(resolved.orgId, qn);
|
|
756
|
+
purgeNodeVector(qn);
|
|
757
|
+
removed += 1;
|
|
758
|
+
}
|
|
759
|
+
});
|
|
760
|
+
deletions += removed;
|
|
761
|
+
if (removed > 0)
|
|
762
|
+
logger.info("live-ingest: ghost sweep removed nodes of vanished members", { removed });
|
|
763
|
+
};
|
|
764
|
+
// Serial parse + apply (incremental deletions).
|
|
765
|
+
const processOne = async (ref, content) => {
|
|
766
|
+
const outcome = ref.obsolete
|
|
767
|
+
? { kind: "skip", parseMs: 0 }
|
|
768
|
+
: await parseMember(ref, content, parseCtxBase, logger);
|
|
769
|
+
applyMember(ref, outcome);
|
|
770
|
+
};
|
|
771
|
+
if (mode === "incremental") {
|
|
772
|
+
const sinceIso = existing?.lastSyncedAt
|
|
773
|
+
? new Date(existing.lastSyncedAt).toISOString()
|
|
774
|
+
: new Date(now - 24 * 3600 * 1000).toISOString();
|
|
775
|
+
// Incremental applies DELETIONS (obsolete members need no content) but
|
|
776
|
+
// does NOT re-parse changed/added members: their content is not fetched
|
|
777
|
+
// here, and calling processOne(ref, "") would overwrite real node
|
|
778
|
+
// attributes with empty-parse garbage and emit no edges — actively
|
|
779
|
+
// degrading the graph. Until per-member content fetch lands, changed
|
|
780
|
+
// members are counted and the user is told to run a full ingest to
|
|
781
|
+
// refresh them. (Deletions are still valuable and safe.)
|
|
782
|
+
let changedNotApplied = 0;
|
|
783
|
+
for await (const ref of iterChanges(resolved.conn, resolved.orgId, sinceIso)) {
|
|
784
|
+
if (ref.obsolete) {
|
|
785
|
+
await processOne(ref, "");
|
|
471
786
|
}
|
|
472
787
|
else {
|
|
473
|
-
|
|
474
|
-
}
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
788
|
+
changedNotApplied += 1;
|
|
789
|
+
}
|
|
790
|
+
}
|
|
791
|
+
incrementalUnfetched = changedNotApplied;
|
|
792
|
+
if (changedNotApplied > 0) {
|
|
793
|
+
const msg = `incremental: ${changedNotApplied} changed/added member(s) detected but NOT refreshed (content fetch not implemented for incremental). The sync watermark is being HELD so the next run re-detects them. Run \`sfgraph ingest --org ${resolved.alias} --rebuild\` (or --mode full) to capture them.`;
|
|
794
|
+
logger.warn(msg);
|
|
795
|
+
wedgeWarnings.push(`incremental:contentNotFetched:count=${changedNotApplied}`);
|
|
796
|
+
}
|
|
797
|
+
}
|
|
798
|
+
else {
|
|
799
|
+
// The outer iteration is also wrapped: bulkRetrieve uses fail-soft per
|
|
800
|
+
// source, but a top-level catch protects against any iterator-protocol
|
|
801
|
+
// surprises (e.g. generator throws during `next()` before yielding).
|
|
802
|
+
// Read live pool caps so the log reflects --tooling-pool/--metadata-pool/
|
|
803
|
+
// --data-pool overrides or SFGRAPH_*_POOL env vars rather than hardcoded
|
|
804
|
+
// defaults.
|
|
805
|
+
const { toolingPool, metadataPool, dataPool } = await import("../extractors/live-org/rate-limit.js");
|
|
806
|
+
const tCap = toolingPool._store.storeOptions.maxConcurrent;
|
|
807
|
+
const mCap = metadataPool._store.storeOptions.maxConcurrent;
|
|
808
|
+
const dCap = dataPool
|
|
809
|
+
._store.storeOptions.maxConcurrent;
|
|
810
|
+
console.log(`ingest: starting full sync (Tooling pool ${tCap} / Metadata pool ${mCap} / Data pool ${dCap} concurrent)`);
|
|
811
|
+
// MCD baseline runs BEFORE the parser fan-out so any (src, dst, REFERENCES)
|
|
812
|
+
// edges it writes can be overwritten if a real parser produces the same
|
|
813
|
+
// shape later. The MCD path uses generic REFERENCES while parsers use
|
|
814
|
+
// specific rel-types — so in practice MCD and parsed coexist in
|
|
815
|
+
// different edge tables rather than competing for the same row, but
|
|
816
|
+
// the pre-fan-out ordering preserves the "parsed wins" guarantee for
|
|
817
|
+
// the (rare) case where a parser does emit REFERENCES.
|
|
818
|
+
openBulk();
|
|
819
|
+
if (!opts.disableMcdBaseline && !opts.metadataSource) {
|
|
820
|
+
try {
|
|
821
|
+
const { runMcdBaseline } = await import("../extractors/live-org/extractors/mcd-baseline.js");
|
|
822
|
+
const mcdCtx = {
|
|
823
|
+
...parseCtxBase,
|
|
824
|
+
sourceUri: "mcd-baseline://tooling-soql",
|
|
825
|
+
};
|
|
826
|
+
const mcd = await runMcdBaseline(resolved.conn, {
|
|
827
|
+
orgId: resolved.orgId,
|
|
828
|
+
ctx: mcdCtx,
|
|
829
|
+
onError: (label, err) => {
|
|
830
|
+
skipReport.skips.push({
|
|
831
|
+
label,
|
|
832
|
+
reason: err.message,
|
|
833
|
+
category: "unknown",
|
|
834
|
+
});
|
|
835
|
+
},
|
|
836
|
+
});
|
|
837
|
+
if (mcd.edges.length > 0) {
|
|
838
|
+
graph.mergeEdges(mcd.edges);
|
|
493
839
|
}
|
|
840
|
+
logger.info("live-ingest: MCD baseline complete", {
|
|
841
|
+
edges: mcd.edges.length,
|
|
842
|
+
byType: mcd.byType,
|
|
843
|
+
});
|
|
844
|
+
}
|
|
845
|
+
catch (e) {
|
|
846
|
+
logger.warn("live-ingest: MCD baseline failed", { err: e.message });
|
|
494
847
|
}
|
|
495
|
-
}, 10_000);
|
|
496
|
-
if (debug) {
|
|
497
|
-
console.log("ingest: [debug] SFGRAPH_DEBUG_INGEST=1 active — heartbeat every 10s");
|
|
498
|
-
// Signal handlers — surface what was running when the user (or OS)
|
|
499
|
-
// sends a kill so silent terminations have at least one breadcrumb.
|
|
500
|
-
const onSig = (sig) => () => {
|
|
501
|
-
console.error(`ingest: [debug] received ${sig} after ${Math.round((Date.now() - fanOutStart) / 1000)}s — processed=${progressCount} lastSource=${lastMemberLabel}`);
|
|
502
|
-
console.error(new Error(`signal:${sig}`).stack);
|
|
503
|
-
process.exitCode = 130;
|
|
504
|
-
};
|
|
505
|
-
const onUsr2 = () => {
|
|
506
|
-
const mem = process.memoryUsage();
|
|
507
|
-
console.error(`ingest: [debug] SIGUSR2 — heap=${Math.round(mem.heapUsed / 1024 / 1024)}MB rss=${Math.round(mem.rss / 1024 / 1024)}MB lastSource=${lastMemberLabel}`);
|
|
508
|
-
};
|
|
509
|
-
installedSignalHandlers.push({ sig: "SIGTERM", fn: onSig("SIGTERM") });
|
|
510
|
-
installedSignalHandlers.push({ sig: "SIGINT", fn: onSig("SIGINT") });
|
|
511
|
-
installedSignalHandlers.push({ sig: "SIGUSR2", fn: onUsr2 });
|
|
512
|
-
for (const h of installedSignalHandlers)
|
|
513
|
-
process.on(h.sig, h.fn);
|
|
514
848
|
}
|
|
849
|
+
const fanOutStart = Date.now();
|
|
850
|
+
let progressCount = 0;
|
|
851
|
+
let lastTickAt = Date.now();
|
|
852
|
+
let lastActivityAt = Date.now();
|
|
853
|
+
let lastMemberLabel = "(none)";
|
|
854
|
+
const PROGRESS_TICK_MS = 5000; // emit a heartbeat at least every 5s
|
|
855
|
+
if (opts.onlyLabels && opts.onlyLabels.size > 0) {
|
|
856
|
+
console.log(`ingest: --only filter active (${opts.onlyLabels.size} source${opts.onlyLabels.size === 1 ? "" : "s"}): ${[...opts.onlyLabels].join(", ")}`);
|
|
857
|
+
}
|
|
858
|
+
// Debug mode: SFGRAPH_DEBUG_INGEST=1 enables a heartbeat timer that
|
|
859
|
+
// prints heap usage + seconds-since-last-activity every 10s. Critical
|
|
860
|
+
// for diagnosing silent ingest deaths (OOM, native segfault, hung SF
|
|
861
|
+
// call) where the normal progress log just stops without an error.
|
|
862
|
+
const debug = process.env.SFGRAPH_DEBUG_INGEST === "1";
|
|
863
|
+
let heartbeatTimer = null;
|
|
864
|
+
let keepAlive = null;
|
|
865
|
+
let livenessProbe = null;
|
|
866
|
+
// Track signal handlers we register so we can remove them in finally.
|
|
867
|
+
// Without this, every invocation of liveIngest in debug mode leaked one
|
|
868
|
+
// listener per signal (SIGTERM/SIGINT/SIGUSR2), eventually triggering
|
|
869
|
+
// MaxListenersExceededWarning and N-fold handler firing on first ^C in
|
|
870
|
+
// multi-org / programmatic flows. (Audit finding C2.)
|
|
871
|
+
const installedSignalHandlers = [];
|
|
872
|
+
// streamAborted (hoisted at function scope) gates the post-fan-out
|
|
873
|
+
// detect-deletions block. If the bulkRetrieve stream itself throws
|
|
874
|
+
// (vs individual sources fail-softing), we've only seen a fraction of
|
|
875
|
+
// the org and the `touchedQnames` set is not safe to use for stale-
|
|
876
|
+
// qname computation — without this guard a transient SF error would
|
|
877
|
+
// mass-delete the untouched 70% of the graph. (Audit finding C1.)
|
|
515
878
|
try {
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
879
|
+
// Keep-alive sentinel: a ref'd timer that exists purely to prevent the
|
|
880
|
+
// event loop from draining while we're awaiting Bottleneck-scheduled
|
|
881
|
+
// work. Bottleneck's local-datastore reservoir-refresh timer is unref'd
|
|
882
|
+
// internally, jsforce's HTTP keep-alive sockets idle between requests,
|
|
883
|
+
// and Node will happily exit(0) on a perfectly healthy pending Promise
|
|
884
|
+
// if no ref'd handle is left.
|
|
885
|
+
keepAlive = setInterval(() => { }, 60_000);
|
|
886
|
+
// Background liveness probe. Polls conn.identity() every 30s with a
|
|
887
|
+
// 10s deadline; after two consecutive failures it logs a prominent
|
|
888
|
+
// "CONNECTION LOST" warning so the user knows why surviving
|
|
889
|
+
// extractors are about to time out one by one. We intentionally do
|
|
890
|
+
// NOT abort the ingest from here — per-call 60s timeouts (added in
|
|
891
|
+
// 1.1.3) already cap exposure, and aborting mid-flight would need
|
|
892
|
+
// an AbortSignal threaded through every extractor for marginal
|
|
893
|
+
// benefit. The probe's job is observability, not control flow.
|
|
894
|
+
// Disable via SFGRAPH_NO_LIVENESS_PROBE=1 for hermetic test runs
|
|
895
|
+
// where the mock connection has no identity() implementation.
|
|
896
|
+
if (process.env.SFGRAPH_NO_LIVENESS_PROBE !== "1" && !opts.metadataSource) {
|
|
897
|
+
livenessProbe = startLivenessProbe(resolved.conn);
|
|
898
|
+
}
|
|
899
|
+
// Wall-clock heartbeat — always on. Without this, silent phases (where
|
|
900
|
+
// all in-window sources are awaiting Bottleneck-queued metadata reads
|
|
901
|
+
// without yielding) look indistinguishable from a hung process. The
|
|
902
|
+
// per-record progress tick at the bottom of the fan-out loop only
|
|
903
|
+
// fires when a record arrives; this fires on a real timer regardless.
|
|
904
|
+
// Debug mode adds heap/rss; baseline shows processed/lastSource/idle.
|
|
905
|
+
const showPoolCounters = debug || process.env.SFGRAPH_DEBUG_POOLS === "1";
|
|
906
|
+
heartbeatTimer = setInterval(() => {
|
|
907
|
+
const idleSec = Math.round((Date.now() - lastActivityAt) / 1000);
|
|
908
|
+
if (idleSec < 10)
|
|
909
|
+
return; // suppress chatter when records are flowing
|
|
910
|
+
if (debug) {
|
|
911
|
+
const mem = process.memoryUsage();
|
|
912
|
+
const heapMb = Math.round(mem.heapUsed / 1024 / 1024);
|
|
913
|
+
const rssMb = Math.round(mem.rss / 1024 / 1024);
|
|
914
|
+
const externalMb = Math.round(mem.external / 1024 / 1024);
|
|
915
|
+
console.log(`ingest: [heartbeat] processed=${progressCount} lastSource=${lastMemberLabel} idle=${idleSec}s heap=${heapMb}MB rss=${rssMb}MB ext=${externalMb}MB`);
|
|
530
916
|
}
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
logger.warn("live-ingest: processOne failed", {
|
|
534
|
-
qname: `${member.ref.memberType}:${member.ref.memberName}`,
|
|
535
|
-
error: e.message,
|
|
536
|
-
});
|
|
917
|
+
else {
|
|
918
|
+
console.log(`ingest: …still working — processed=${progressCount} idle=${idleSec}s lastSource=${lastMemberLabel}`);
|
|
537
919
|
}
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
920
|
+
// Pool diagnostics: snapshot Bottleneck state so a wedge can be
|
|
921
|
+
// attributed concretely to reservoir starvation vs. stuck-in-flight
|
|
922
|
+
// jobs vs. pre-yield iterator hangs. Counts are synchronous; the
|
|
923
|
+
// reservoir read is async but fire-and-forget — by the time it
|
|
924
|
+
// resolves we just log it.
|
|
925
|
+
if (showPoolCounters) {
|
|
926
|
+
const snapshot = [
|
|
927
|
+
["tool", toolingPool],
|
|
928
|
+
["meta", metadataPool],
|
|
929
|
+
["data", dataPool],
|
|
930
|
+
];
|
|
931
|
+
for (const [name, p] of snapshot) {
|
|
932
|
+
const c = p.counts();
|
|
933
|
+
p.currentReservoir().then((reservoir) => {
|
|
934
|
+
console.log(`ingest: [pool ${name}] running=${c.RUNNING} executing=${c.EXECUTING} queued=${c.QUEUED} received=${c.RECEIVED} reservoir=${reservoir}`);
|
|
935
|
+
}, () => {
|
|
936
|
+
/* swallow — never let a diag failure kill ingest */
|
|
937
|
+
});
|
|
938
|
+
}
|
|
548
939
|
}
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
940
|
+
}, 10_000);
|
|
941
|
+
if (debug) {
|
|
942
|
+
console.log("ingest: [debug] SFGRAPH_DEBUG_INGEST=1 active — heartbeat every 10s");
|
|
943
|
+
// Signal handlers — surface what was running when the user (or OS)
|
|
944
|
+
// sends a kill so silent terminations have at least one breadcrumb.
|
|
945
|
+
const onSig = (sig) => () => {
|
|
946
|
+
console.error(`ingest: [debug] received ${sig} after ${Math.round((Date.now() - fanOutStart) / 1000)}s — processed=${progressCount} lastSource=${lastMemberLabel}`);
|
|
947
|
+
console.error(new Error(`signal:${sig}`).stack);
|
|
948
|
+
process.exitCode = 130;
|
|
949
|
+
// rel-08: a breadcrumb only — never swallow the signal. Drop the
|
|
950
|
+
// debug handlers; if nothing else handles it (an embedding host
|
|
951
|
+
// or the CLI would), re-raise so the default action terminates.
|
|
952
|
+
// Review minor 12: on the library path the default action kills
|
|
953
|
+
// the process without the finally blocks, so the ingest lock file
|
|
954
|
+
// and sync_in_progress stay behind; the next run reclaims both
|
|
955
|
+
// through the dead-owner (stale pid) check (rel-02).
|
|
956
|
+
for (const h of installedSignalHandlers.splice(0))
|
|
957
|
+
process.off(h.sig, h.fn);
|
|
958
|
+
if (process.listenerCount(sig) === 0)
|
|
959
|
+
process.kill(process.pid, sig);
|
|
960
|
+
};
|
|
961
|
+
const onUsr2 = () => {
|
|
962
|
+
const mem = process.memoryUsage();
|
|
963
|
+
console.error(`ingest: [debug] SIGUSR2 — heap=${Math.round(mem.heapUsed / 1024 / 1024)}MB rss=${Math.round(mem.rss / 1024 / 1024)}MB lastSource=${lastMemberLabel}`);
|
|
964
|
+
};
|
|
965
|
+
installedSignalHandlers.push({ sig: "SIGTERM", fn: onSig("SIGTERM") });
|
|
966
|
+
installedSignalHandlers.push({ sig: "SIGINT", fn: onSig("SIGINT") });
|
|
967
|
+
installedSignalHandlers.push({ sig: "SIGUSR2", fn: onUsr2 });
|
|
968
|
+
for (const h of installedSignalHandlers)
|
|
969
|
+
process.on(h.sig, h.fn);
|
|
970
|
+
}
|
|
971
|
+
try {
|
|
972
|
+
const memberStream = deltaHooks
|
|
973
|
+
? deltaHooks.members
|
|
974
|
+
: opts.metadataSource
|
|
975
|
+
? opts.metadataSource.iter()
|
|
976
|
+
: bulkRetrieve(resolved.conn, caps, resolved.orgId, {
|
|
977
|
+
skipReport,
|
|
978
|
+
warnings: wedgeWarnings,
|
|
979
|
+
...(opts.onlyLabels ? { onlyLabels: opts.onlyLabels } : {}),
|
|
980
|
+
...(opts.explicitOnlyLabels ? { explicitOnlyLabels: true } : {}),
|
|
981
|
+
// The bulk-retrieve flag name follows the same convention as
|
|
982
|
+
// the bulk-retrieve's other opts; semantically it's "do the
|
|
983
|
+
// retrieve()" which is now on by default. Caller-side disable
|
|
984
|
+
// routes through this same boolean inverted.
|
|
985
|
+
...(opts.disableOmnistudioRetrieve ? {} : { enableOmnistudioRetrieve: true }),
|
|
986
|
+
// jsforce attaches the version it negotiated on conn.version; fall
|
|
987
|
+
// back to a sensible recent release if absent (mocks, etc.)
|
|
988
|
+
apiVersion: resolved.conn.version ?? "60.0",
|
|
989
|
+
});
|
|
990
|
+
const tFanout = performance.now();
|
|
991
|
+
// Task 11b: parse in worker threads (SFGRAPH_PARSE_WORKERS, default
|
|
992
|
+
// min(4, availableParallelism - 1); 0 = main thread), apply in member
|
|
993
|
+
// order here.
|
|
994
|
+
const parseStats = opts.parsePool?.stats ?? {
|
|
995
|
+
workers: 0,
|
|
996
|
+
pooled: 0,
|
|
997
|
+
inProcess: 0,
|
|
998
|
+
fallbacks: 0,
|
|
999
|
+
crashes: 0,
|
|
1000
|
+
oomFallbacks: 0,
|
|
1001
|
+
};
|
|
1002
|
+
const parsed = parsePool(memberStream, {
|
|
1003
|
+
orgId: parseCtxBase.orgId,
|
|
1004
|
+
parseTimestamp: parseCtxBase.parseTimestamp,
|
|
1005
|
+
logger,
|
|
1006
|
+
workers: opts.parsePool?.workers ?? resolveParseWorkers(),
|
|
1007
|
+
...(opts.parsePool?.workerUrl !== undefined
|
|
1008
|
+
? { workerUrl: opts.parsePool.workerUrl }
|
|
1009
|
+
: {}),
|
|
1010
|
+
...(opts.parsePool?.minMembers !== undefined
|
|
1011
|
+
? { minMembers: opts.parsePool.minMembers }
|
|
1012
|
+
: {}),
|
|
1013
|
+
stats: parseStats,
|
|
1014
|
+
...(quarantinedKeys.size > 0 ? { preset: quarantinePreset } : {}),
|
|
1015
|
+
});
|
|
1016
|
+
for await (const { member, outcome } of parsed) {
|
|
1017
|
+
const errorsBefore = parseErrors;
|
|
1018
|
+
if (batchOpen)
|
|
1019
|
+
graph.batchMemberStart?.();
|
|
1020
|
+
try {
|
|
1021
|
+
applyMember(member.ref, outcome);
|
|
1022
|
+
if (profiler.enabled) {
|
|
1023
|
+
profiler.member({
|
|
1024
|
+
qname: `${member.ref.memberType}:${member.ref.memberName}`,
|
|
1025
|
+
label: member.ref.memberType,
|
|
1026
|
+
bytes: member.content.length,
|
|
1027
|
+
parseMs: lastParseMs,
|
|
1028
|
+
writeMs: lastWriteMs,
|
|
1029
|
+
});
|
|
1030
|
+
}
|
|
1031
|
+
}
|
|
1032
|
+
catch (e) {
|
|
1033
|
+
parseErrors += 1;
|
|
1034
|
+
logger.warn("live-ingest: processOne failed", {
|
|
1035
|
+
qname: `${member.ref.memberType}:${member.ref.memberName}`,
|
|
1036
|
+
error: e.message,
|
|
1037
|
+
});
|
|
1038
|
+
}
|
|
1039
|
+
// Task 11c: member rows (what each member owns) for delta ingest.
|
|
1040
|
+
const errored = outcome.kind === "error" || parseErrors > errorsBefore;
|
|
1041
|
+
if (deltaHooks)
|
|
1042
|
+
deltaHooks.onMember(member, outcome, errored);
|
|
1043
|
+
else if (memberRows && member.fingerprint !== undefined) {
|
|
1044
|
+
recordMemberRow(member, outcome, errored);
|
|
1045
|
+
}
|
|
1046
|
+
if (bulkOpen || batchOpen) {
|
|
1047
|
+
// Outside the per-member catch: if SQLite rolled back the bulk
|
|
1048
|
+
// transaction (SQLITE_FULL / IOERR / NOMEM), up to 2,000 earlier
|
|
1049
|
+
// members are gone, so the ingest must fail, not count one error.
|
|
1050
|
+
// The same holds for a live-graph batch.
|
|
1051
|
+
try {
|
|
1052
|
+
if (bulkOpen)
|
|
1053
|
+
graph.bulkMemberDone?.();
|
|
1054
|
+
else
|
|
1055
|
+
graph.batchMemberDone?.();
|
|
1056
|
+
}
|
|
1057
|
+
catch (e) {
|
|
1058
|
+
bulkFailure = e;
|
|
1059
|
+
break;
|
|
1060
|
+
}
|
|
1061
|
+
}
|
|
1062
|
+
// Backpressure: past the queued-text bound, wait for forced
|
|
1063
|
+
// batches to drain (the only inference during fan-out).
|
|
1064
|
+
if (embedQueue?.overCapacity)
|
|
1065
|
+
await embedQueue.waitForCapacity();
|
|
1066
|
+
progressCount += 1;
|
|
1067
|
+
lastActivityAt = Date.now();
|
|
1068
|
+
lastMemberLabel = `${member.ref.memberType}:${member.ref.memberName}`;
|
|
1069
|
+
if (progressCount % 200 === 0 || Date.now() - lastTickAt > PROGRESS_TICK_MS) {
|
|
1070
|
+
const elapsedSec = Math.round((Date.now() - fanOutStart) / 1000);
|
|
1071
|
+
const memSuffix = debug
|
|
1072
|
+
? ` heap=${Math.round(process.memoryUsage().heapUsed / 1024 / 1024)}MB`
|
|
1073
|
+
: "";
|
|
1074
|
+
console.log(`ingest: …${progressCount} members processed so far (${elapsedSec}s elapsed)${memSuffix}`);
|
|
1075
|
+
lastTickAt = Date.now();
|
|
1076
|
+
}
|
|
1077
|
+
if (progressCount % 500 === 0) {
|
|
1078
|
+
const gs = graph;
|
|
1079
|
+
if (typeof gs.checkpoint === "function")
|
|
1080
|
+
gs.checkpoint();
|
|
1081
|
+
}
|
|
553
1082
|
}
|
|
1083
|
+
profiler.phase("fanout", performance.now() - tFanout);
|
|
1084
|
+
// Always printed (serial too) so bench runs can record the counts.
|
|
1085
|
+
console.log(`ingest: parse pool: ${parseStats.workers} workers, ${parseStats.pooled} parsed in workers, ${parseStats.inProcess} on main (${parseStats.fallbacks} fallbacks, ${parseStats.crashes} worker crashes, ${parseStats.oomFallbacks ?? 0} heap-cap re-parses)`);
|
|
1086
|
+
console.log(`ingest: fan-out complete (${progressCount} members in ${Math.round((Date.now() - fanOutStart) / 1000)}s)`);
|
|
1087
|
+
const gs = graph;
|
|
1088
|
+
if (typeof gs.checkpoint === "function")
|
|
1089
|
+
gs.checkpoint();
|
|
1090
|
+
}
|
|
1091
|
+
catch (e) {
|
|
1092
|
+
// The stream itself threw (not an individual source — those are
|
|
1093
|
+
// captured by failSoft). Mark so detect-deletions stays its hand.
|
|
1094
|
+
streamAborted = true;
|
|
1095
|
+
logger.warn("live-ingest: bulkRetrieve stream aborted", {
|
|
1096
|
+
error: e.message,
|
|
1097
|
+
});
|
|
1098
|
+
// Never a silent partial ingest: the abort is in the result's warnings.
|
|
1099
|
+
wedgeWarnings.push(`stream:aborted:${String(e.message).slice(0, 200)}`);
|
|
554
1100
|
}
|
|
555
|
-
console.log(`ingest: fan-out complete (${progressCount} members in ${Math.round((Date.now() - fanOutStart) / 1000)}s)`);
|
|
556
|
-
const gs = graph;
|
|
557
|
-
if (typeof gs.checkpoint === "function")
|
|
558
|
-
gs.checkpoint();
|
|
559
1101
|
}
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
1102
|
+
finally {
|
|
1103
|
+
if (heartbeatTimer)
|
|
1104
|
+
clearInterval(heartbeatTimer);
|
|
1105
|
+
if (keepAlive)
|
|
1106
|
+
clearInterval(keepAlive);
|
|
1107
|
+
livenessProbe?.stop();
|
|
1108
|
+
for (const h of installedSignalHandlers)
|
|
1109
|
+
process.off(h.sig, h.fn);
|
|
1110
|
+
// Commit the last bulk batch and rebuild the deferred indexes before
|
|
1111
|
+
// any post-pass reads the graph.
|
|
1112
|
+
const tEndBulk = performance.now();
|
|
1113
|
+
closeBulk();
|
|
1114
|
+
if (graphEmptyAtStart)
|
|
1115
|
+
profiler.phase("endBulk", performance.now() - tEndBulk);
|
|
1116
|
+
}
|
|
1117
|
+
if (bulkFailure)
|
|
1118
|
+
throw bulkFailure;
|
|
1119
|
+
if (deltaHooks) {
|
|
1120
|
+
// A partial stream must not be finalized as a delta: fail, keeping
|
|
1121
|
+
// the pending marker, so the next delta replays it.
|
|
1122
|
+
if (streamAborted)
|
|
1123
|
+
throw new Error("delta: member stream aborted");
|
|
1124
|
+
deltaHooks.afterMerge();
|
|
1125
|
+
}
|
|
1126
|
+
else if (memberRows) {
|
|
1127
|
+
finishMemberRows();
|
|
1128
|
+
}
|
|
1129
|
+
// A scoped run (auto-retry pass, --retry-skipped, --only) re-ran only
|
|
1130
|
+
// `onlyLabels`: carry the previous report's entries for every OTHER
|
|
1131
|
+
// label forward, so unreadable / insufficient_access entries it did
|
|
1132
|
+
// not touch stay in the report, the summary and the warnings (I2).
|
|
1133
|
+
if (opts.onlyLabels && opts.skipReportPath) {
|
|
1134
|
+
skipReport.skips.push(...carriedOverSkips(opts.skipReportPath, resolved.orgId, opts.onlyLabels));
|
|
1135
|
+
}
|
|
1136
|
+
// End-of-run skip summary. Grouped by category so the remediation is
|
|
1137
|
+
// targeted to the specific class of failure.
|
|
1138
|
+
if (skipReport.skips.length > 0) {
|
|
1139
|
+
printSkipSummary(skipReport, resolved.alias);
|
|
1140
|
+
}
|
|
1141
|
+
// Persist the skip report so --retry-skipped can read it next run.
|
|
1142
|
+
// Always written (even when empty) so users can tell whether the last
|
|
1143
|
+
// run had skips or just hasn't recorded any yet.
|
|
1144
|
+
if (opts.skipReportPath) {
|
|
1145
|
+
try {
|
|
1146
|
+
const { writeFileSync, mkdirSync } = await import("node:fs");
|
|
1147
|
+
const { dirname } = await import("node:path");
|
|
1148
|
+
mkdirSync(dirname(opts.skipReportPath), { recursive: true });
|
|
1149
|
+
writeFileSync(opts.skipReportPath, JSON.stringify({ recordedAt: Date.now(), orgId: resolved.orgId, ...skipReport }, null, 2), "utf8");
|
|
1150
|
+
}
|
|
1151
|
+
catch (e) {
|
|
1152
|
+
logger.warn("live-ingest: skip report write failed", { err: e.message });
|
|
1153
|
+
}
|
|
567
1154
|
}
|
|
568
1155
|
}
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
1156
|
+
// Mark the end of input (the held queue still runs nothing until graph ready).
|
|
1157
|
+
embedQueue?.startDrain();
|
|
1158
|
+
// Full-sync deletion detection: compute previously-known minus touched and
|
|
1159
|
+
// delete the difference. Skipped on incremental (SourceMember handles it),
|
|
1160
|
+
// skipped if any parse error occurred this run (avoid mass-wipe on
|
|
1161
|
+
// transient SF errors).
|
|
1162
|
+
// Fast path: an org that was empty at start has no stale qnames — every
|
|
1163
|
+
// persisted node was touched by this run — so the sweep (and its
|
|
1164
|
+
// deleteEdgesFor calls) would be a no-op; skip it.
|
|
1165
|
+
if (opts.detectDeletions && !opts.metadataSource && mode === "full" && !graphEmptyAtStart) {
|
|
1166
|
+
if (streamAborted) {
|
|
1167
|
+
logger.warn("live-ingest: detect-deletions skipped because bulkRetrieve stream aborted — touchedQnames is incomplete and a full deletion sweep would wrongly wipe untouched-but-still-present nodes");
|
|
1168
|
+
}
|
|
1169
|
+
else if (parseErrors > 0) {
|
|
1170
|
+
logger.warn("live-ingest: detect-deletions skipped due to parseErrors", { parseErrors });
|
|
1171
|
+
}
|
|
1172
|
+
else {
|
|
1173
|
+
try {
|
|
1174
|
+
// W1.5-07: per-label drop-ratio guard. The sweep used to run as a
|
|
1175
|
+
// single set-difference across ALL labels; a wedged source with
|
|
1176
|
+
// streamAborted=false + touchedCount=0 for its label would silently
|
|
1177
|
+
// wipe every node of that label (graph-extinction risk that
|
|
1178
|
+
// compounds with the W1.5-01/02 wedge cascade). Now we evaluate the
|
|
1179
|
+
// guard per label and refuse when the implied drop is suspicious.
|
|
1180
|
+
const maxDropRatio = resolveMaxDropRatio(process.env.SFGRAPH_DETECT_DELETIONS_MAX_DROP_RATIO);
|
|
1181
|
+
// Task 11c: a node from a source this run skipped (or carried a
|
|
1182
|
+
// skip for) is never "stale" — the source touched nothing.
|
|
1183
|
+
const protection = skipProtection(skipReport.skips);
|
|
1184
|
+
if (protection.refuseAll) {
|
|
1185
|
+
const warning = `wedge:detect-deletions:refuse:label=*:reason=skipped-source:skips=${protection.unknownLabels.join(",")}`;
|
|
1186
|
+
wedgeWarnings.push(warning);
|
|
1187
|
+
logger.warn("live-ingest: detect-deletions refused (a skipped source has no type mapping)", {
|
|
1188
|
+
skips: protection.unknownLabels,
|
|
1189
|
+
});
|
|
1190
|
+
}
|
|
1191
|
+
const labels = protection.refuseAll ? [] : graph.listAllLabels();
|
|
1192
|
+
let protectedCount = 0;
|
|
1193
|
+
for (const label of labels) {
|
|
1194
|
+
const priorCount = graph.countNodesByLabel(resolved.orgId, label);
|
|
1195
|
+
const touchedCount = touchedQnamesByLabel.get(label)?.size ?? 0;
|
|
1196
|
+
const verdict = evaluateDeletionGuard(label, priorCount, touchedCount, maxDropRatio);
|
|
1197
|
+
if (!verdict.proceed) {
|
|
1198
|
+
if (verdict.warning)
|
|
1199
|
+
wedgeWarnings.push(verdict.warning);
|
|
1200
|
+
logger.warn("live-ingest: detect-deletions refused", {
|
|
1201
|
+
label,
|
|
1202
|
+
priorCount,
|
|
1203
|
+
touchedCount,
|
|
1204
|
+
maxDropRatio,
|
|
1205
|
+
warning: verdict.warning,
|
|
1206
|
+
});
|
|
1207
|
+
continue;
|
|
1208
|
+
}
|
|
1209
|
+
// Proceed with deletion for this label only. Scoped to this label's
|
|
1210
|
+
// table via listNodesByLabel rather than the global listAllQnames
|
|
1211
|
+
// walk, so a refusal for one label cannot accidentally affect
|
|
1212
|
+
// sweeps for other labels.
|
|
1213
|
+
const touchedSet = touchedQnamesByLabel.get(label);
|
|
1214
|
+
const persisted = graph.listNodesByLabel(resolved.orgId, label);
|
|
1215
|
+
let labelDeletions = 0;
|
|
1216
|
+
for (const node of persisted) {
|
|
1217
|
+
const q = String(node.qualifiedName);
|
|
1218
|
+
if (touchedSet?.has(q))
|
|
1219
|
+
continue;
|
|
1220
|
+
if (isSkipProtected(protection, node)) {
|
|
1221
|
+
protectedCount += 1;
|
|
1222
|
+
continue;
|
|
1223
|
+
}
|
|
1224
|
+
graph.deleteEdgesFor(resolved.orgId, asQualifiedName(q));
|
|
1225
|
+
graph.deleteNode(resolved.orgId, asQualifiedName(q));
|
|
1226
|
+
purgeNodeVector(asQualifiedName(q));
|
|
1227
|
+
deletions += 1;
|
|
1228
|
+
labelDeletions += 1;
|
|
1229
|
+
}
|
|
1230
|
+
if (labelDeletions > 0) {
|
|
1231
|
+
logger.info("live-ingest: detect-deletions removed stale qnames", {
|
|
1232
|
+
label,
|
|
1233
|
+
count: labelDeletions,
|
|
1234
|
+
});
|
|
1235
|
+
}
|
|
1236
|
+
}
|
|
1237
|
+
if (protectedCount > 0) {
|
|
1238
|
+
logger.info("live-ingest: detect-deletions kept nodes of skipped sources", {
|
|
1239
|
+
count: protectedCount,
|
|
1240
|
+
});
|
|
1241
|
+
}
|
|
1242
|
+
}
|
|
1243
|
+
catch (e) {
|
|
1244
|
+
logger.warn("live-ingest: detect-deletions failed", { err: e.message });
|
|
1245
|
+
}
|
|
1246
|
+
}
|
|
1247
|
+
}
|
|
1248
|
+
// Backfill: an earlier run returned before its vectors were all written
|
|
1249
|
+
// (--no-wait-embeddings, a worker crash, a kill). Every embedded text is
|
|
1250
|
+
// also stored verbatim as the node's FTS body, so re-queue the docs this
|
|
1251
|
+
// run did not touch whose stored vector is missing or stale.
|
|
1252
|
+
let backfilled = 0;
|
|
1253
|
+
if (embedQueue && wasEmbeddingsPending && graph.listFtsDocs) {
|
|
1254
|
+
const sink = opts.vectorStore;
|
|
587
1255
|
try {
|
|
588
|
-
const
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
1256
|
+
for (const doc of graph.listFtsDocs(resolved.orgId)) {
|
|
1257
|
+
if (touchedQnames.has(doc.qname))
|
|
1258
|
+
continue;
|
|
1259
|
+
const stored = sink?.getContentHash?.(resolved.orgId, asQualifiedName(doc.qname));
|
|
1260
|
+
if (stored && stored === hashEmbedText(doc.body))
|
|
1261
|
+
continue;
|
|
1262
|
+
embedQueue.push({
|
|
1263
|
+
qname: doc.qname,
|
|
1264
|
+
text: doc.body,
|
|
1265
|
+
orgId: String(resolved.orgId),
|
|
1266
|
+
label: doc.label,
|
|
1267
|
+
});
|
|
1268
|
+
backfilled += 1;
|
|
1269
|
+
if (embedQueue.overCapacity)
|
|
1270
|
+
await embedQueue.waitForCapacity();
|
|
1271
|
+
}
|
|
592
1272
|
}
|
|
593
1273
|
catch (e) {
|
|
594
|
-
logger.warn("live-ingest:
|
|
1274
|
+
logger.warn("live-ingest: embedding backfill failed", { err: e.message });
|
|
595
1275
|
}
|
|
1276
|
+
if (backfilled > 0) {
|
|
1277
|
+
logger.info("live-ingest: re-queued pending embeddings", { backfilled });
|
|
1278
|
+
}
|
|
1279
|
+
embedQueue.startDrain();
|
|
596
1280
|
}
|
|
597
|
-
|
|
598
|
-
|
|
1281
|
+
// -----------------------------------------------------------------------
|
|
1282
|
+
// Post-merge resolver passes. Each step is isolated in try/catch so a
|
|
1283
|
+
// single resolver bug never breaks ingest — it just reports zero work done.
|
|
1284
|
+
// -----------------------------------------------------------------------
|
|
1285
|
+
let crossFlavorEdges = 0;
|
|
1286
|
+
let arityResolved = 0;
|
|
1287
|
+
let flowMethodsResolved = 0;
|
|
1288
|
+
let vlocityApexResult;
|
|
1289
|
+
let drRelationshipsResult;
|
|
1290
|
+
let danglingEdges = 0;
|
|
1291
|
+
let overlapResult = { matched: 0, diverged: 0, empty: 0, annotated: 0 };
|
|
1292
|
+
let reflectionEdges = 0;
|
|
1293
|
+
// T15 I-2: post-merge steps that threw. A full ingest only warns; a delta
|
|
1294
|
+
// must not finalize (see below).
|
|
1295
|
+
const postFailures = [];
|
|
1296
|
+
// Synth a ParseContext for post-passes that need to mint edges/nodes.
|
|
1297
|
+
const postCtx = {
|
|
1298
|
+
...parseCtxBase,
|
|
1299
|
+
sourceUri: "post-merge://resolver",
|
|
1300
|
+
};
|
|
1301
|
+
// review I4: re-target Aura `<c:name>` embeds at the bundles that exist.
|
|
599
1302
|
try {
|
|
600
|
-
|
|
1303
|
+
const auraEmbeds = resolveAuraEmbeds(graph, resolved.orgId);
|
|
1304
|
+
if (auraEmbeds.retargeted > 0) {
|
|
1305
|
+
logger.info("live-ingest: aura embed resolver re-targeted embeds", { ...auraEmbeds });
|
|
1306
|
+
}
|
|
601
1307
|
}
|
|
602
1308
|
catch (e) {
|
|
603
|
-
|
|
1309
|
+
postFailures.push("aura embed resolver");
|
|
1310
|
+
logger.warn("live-ingest: aura embed resolver failed", { err: e.message });
|
|
604
1311
|
}
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
1312
|
+
if (!opts.disableCrossFlavor) {
|
|
1313
|
+
try {
|
|
1314
|
+
crossFlavorEdges = resolveCrossFlavor(graph, {
|
|
1315
|
+
orgId: resolved.orgId,
|
|
1316
|
+
namespace: null,
|
|
1317
|
+
ctx: postCtx,
|
|
1318
|
+
});
|
|
1319
|
+
if (crossFlavorEdges > 0) {
|
|
1320
|
+
logger.info("live-ingest: cross-flavor resolver linked Vlocity↔OmniStudio", {
|
|
1321
|
+
edges: crossFlavorEdges,
|
|
1322
|
+
});
|
|
1323
|
+
}
|
|
1324
|
+
}
|
|
1325
|
+
catch (e) {
|
|
1326
|
+
postFailures.push("cross-flavor resolver");
|
|
1327
|
+
logger.warn("live-ingest: cross-flavor resolver failed", { err: e.message });
|
|
1328
|
+
}
|
|
613
1329
|
}
|
|
614
|
-
|
|
615
|
-
|
|
1330
|
+
// Overlap detector: opt-in (off by default) — annotates the CANONICAL_OF
|
|
1331
|
+
// edges resolveCrossFlavor just emitted with signaturesMatch + divergence
|
|
1332
|
+
// detail. Must run *after* cross-flavor and *before* the audit so the
|
|
1333
|
+
// audit sees the annotated edges as the same edges (idempotent merge).
|
|
1334
|
+
if (!opts.disableOverlapDetect) {
|
|
1335
|
+
try {
|
|
1336
|
+
const { detectOmnistudioOverlap } = await import("../parsers/omnistudio/overlap-detector.js");
|
|
1337
|
+
overlapResult = detectOmnistudioOverlap(graph, {
|
|
1338
|
+
orgId: resolved.orgId,
|
|
1339
|
+
ctx: postCtx,
|
|
1340
|
+
});
|
|
1341
|
+
if (overlapResult.annotated > 0) {
|
|
1342
|
+
logger.info("live-ingest: omnistudio overlap detector annotated CANONICAL_OF pairs", {
|
|
1343
|
+
matched: overlapResult.matched,
|
|
1344
|
+
diverged: overlapResult.diverged,
|
|
1345
|
+
empty: overlapResult.empty,
|
|
1346
|
+
annotated: overlapResult.annotated,
|
|
1347
|
+
});
|
|
1348
|
+
}
|
|
1349
|
+
}
|
|
1350
|
+
catch (e) {
|
|
1351
|
+
postFailures.push("omnistudio overlap detector");
|
|
1352
|
+
logger.warn("live-ingest: omnistudio overlap detector failed", {
|
|
1353
|
+
err: e.message,
|
|
1354
|
+
});
|
|
1355
|
+
}
|
|
616
1356
|
}
|
|
617
|
-
|
|
1357
|
+
if (!opts.disableFlowInvocableResolve) {
|
|
1358
|
+
const tPhase = performance.now();
|
|
618
1359
|
try {
|
|
619
|
-
const
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
graph.deleteNode(resolved.orgId, asQualifiedName(q));
|
|
628
|
-
deletions += 1;
|
|
629
|
-
}
|
|
630
|
-
if (stale.length > 0) {
|
|
631
|
-
logger.info("live-ingest: detect-deletions removed stale qnames", {
|
|
632
|
-
count: stale.length,
|
|
1360
|
+
const flowResult = resolveFlowApexMethods(graph, resolved.orgId, postCtx);
|
|
1361
|
+
flowMethodsResolved = flowResult.resolved;
|
|
1362
|
+
if (flowResult.scanned > 0) {
|
|
1363
|
+
logger.info("live-ingest: flow→apex invocable resolver", {
|
|
1364
|
+
scanned: flowResult.scanned,
|
|
1365
|
+
resolved: flowResult.resolved,
|
|
1366
|
+
missing: flowResult.missing,
|
|
1367
|
+
ambiguous: flowResult.ambiguous,
|
|
633
1368
|
});
|
|
634
1369
|
}
|
|
635
1370
|
}
|
|
636
1371
|
catch (e) {
|
|
637
|
-
|
|
1372
|
+
postFailures.push("flow→apex resolver");
|
|
1373
|
+
logger.warn("live-ingest: flow→apex resolver failed", { err: e.message });
|
|
638
1374
|
}
|
|
1375
|
+
profiler.phase("resolver:flow-apex", performance.now() - tPhase);
|
|
639
1376
|
}
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
let flowMethodsResolved = 0;
|
|
648
|
-
let danglingEdges = 0;
|
|
649
|
-
let overlapResult = { matched: 0, diverged: 0, empty: 0, annotated: 0 };
|
|
650
|
-
let reflectionEdges = 0;
|
|
651
|
-
// Synth a ParseContext for post-passes that need to mint edges/nodes.
|
|
652
|
-
const postCtx = {
|
|
653
|
-
...parseCtxBase,
|
|
654
|
-
sourceUri: "post-merge://resolver",
|
|
655
|
-
};
|
|
656
|
-
if (!opts.disableCrossFlavor) {
|
|
657
|
-
try {
|
|
658
|
-
crossFlavorEdges = resolveCrossFlavor(graph, {
|
|
659
|
-
orgId: resolved.orgId,
|
|
660
|
-
namespace: null,
|
|
661
|
-
ctx: postCtx,
|
|
662
|
-
});
|
|
663
|
-
if (crossFlavorEdges > 0) {
|
|
664
|
-
logger.info("live-ingest: cross-flavor resolver linked Vlocity↔OmniStudio", {
|
|
665
|
-
edges: crossFlavorEdges,
|
|
666
|
-
});
|
|
1377
|
+
if (!opts.disableVlocityApexResolve) {
|
|
1378
|
+
const tPhase = performance.now();
|
|
1379
|
+
try {
|
|
1380
|
+
const { resolveVlocityApex } = await import("../parsers/vlocity/apex-resolver.js");
|
|
1381
|
+
vlocityApexResult = resolveVlocityApex(graph, resolved.orgId, postCtx);
|
|
1382
|
+
if (vlocityApexResult.scanned > 0)
|
|
1383
|
+
logger.info("live-ingest: vlocity→apex resolver", vlocityApexResult);
|
|
667
1384
|
}
|
|
1385
|
+
catch (e) {
|
|
1386
|
+
postFailures.push("vlocity-apex-resolver");
|
|
1387
|
+
logger.warn(`vlocity-apex-resolver:failed:${e.message}`);
|
|
1388
|
+
wedgeWarnings.push(`vlocity-apex-resolver:failed:${e.message}`);
|
|
1389
|
+
}
|
|
1390
|
+
profiler.phase("resolver:vlocity-apex", performance.now() - tPhase);
|
|
668
1391
|
}
|
|
669
|
-
|
|
670
|
-
|
|
1392
|
+
if (!opts.disableDrRelationshipResolve) {
|
|
1393
|
+
const tPhase = performance.now();
|
|
1394
|
+
try {
|
|
1395
|
+
const { resolveDataRaptorRelationships } = await import("../parsers/vlocity/dr-relationship-resolver.js");
|
|
1396
|
+
drRelationshipsResult = resolveDataRaptorRelationships(graph, resolved.orgId, postCtx);
|
|
1397
|
+
if (drRelationshipsResult.scanned > 0)
|
|
1398
|
+
logger.info("live-ingest: dataraptor relationship-path resolver", drRelationshipsResult);
|
|
1399
|
+
}
|
|
1400
|
+
catch (e) {
|
|
1401
|
+
postFailures.push("dr-relationship-resolver");
|
|
1402
|
+
logger.warn(`dr-relationship-resolver:failed:${e.message}`);
|
|
1403
|
+
wedgeWarnings.push(`dr-relationship-resolver:failed:${e.message}`);
|
|
1404
|
+
}
|
|
1405
|
+
profiler.phase("resolver:dr-relationship", performance.now() - tPhase);
|
|
671
1406
|
}
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
const { detectOmnistudioOverlap } = await import("../parsers/omnistudio/overlap-detector.js");
|
|
680
|
-
overlapResult = detectOmnistudioOverlap(graph, {
|
|
681
|
-
orgId: resolved.orgId,
|
|
682
|
-
ctx: postCtx,
|
|
683
|
-
});
|
|
684
|
-
if (overlapResult.annotated > 0) {
|
|
685
|
-
logger.info("live-ingest: omnistudio overlap detector annotated CANONICAL_OF pairs", {
|
|
686
|
-
matched: overlapResult.matched,
|
|
687
|
-
diverged: overlapResult.diverged,
|
|
688
|
-
empty: overlapResult.empty,
|
|
689
|
-
annotated: overlapResult.annotated,
|
|
690
|
-
});
|
|
1407
|
+
if (!opts.disableFlowRelationshipResolve) {
|
|
1408
|
+
const tPhase = performance.now();
|
|
1409
|
+
try {
|
|
1410
|
+
const { resolveFlowRelationshipPaths } = await import("../parsers/flow/relationship-resolver.js");
|
|
1411
|
+
const fr = resolveFlowRelationshipPaths(graph, resolved.orgId, postCtx);
|
|
1412
|
+
if (fr.scanned > 0)
|
|
1413
|
+
logger.info("live-ingest: flow relationship-path resolver", { ...fr });
|
|
691
1414
|
}
|
|
1415
|
+
catch (e) {
|
|
1416
|
+
postFailures.push("flow-relationship-resolver");
|
|
1417
|
+
logger.warn(`flow-relationship-resolver:failed:${e.message}`);
|
|
1418
|
+
wedgeWarnings.push(`flow-relationship-resolver:failed:${e.message}`);
|
|
1419
|
+
}
|
|
1420
|
+
profiler.phase("resolver:flow-relationship", performance.now() - tPhase);
|
|
692
1421
|
}
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
1422
|
+
if (!opts.disableLwcFieldKeyResolve) {
|
|
1423
|
+
const tPhase = performance.now();
|
|
1424
|
+
try {
|
|
1425
|
+
const { resolveLwcFieldKeys } = await import("../parsers/lwc/field-key-resolver.js");
|
|
1426
|
+
const r = resolveLwcFieldKeys(graph, resolved.orgId, postCtx);
|
|
1427
|
+
if (r.scanned > 0)
|
|
1428
|
+
logger.info("live-ingest: lwc field-key resolver", { ...r });
|
|
1429
|
+
}
|
|
1430
|
+
catch (e) {
|
|
1431
|
+
postFailures.push("lwc-field-key-resolver");
|
|
1432
|
+
logger.warn(`lwc-field-key-resolver:failed:${e.message}`);
|
|
1433
|
+
wedgeWarnings.push(`lwc-field-key-resolver:failed:${e.message}`);
|
|
1434
|
+
}
|
|
1435
|
+
profiler.phase("resolver:lwc-field-key", performance.now() - tPhase);
|
|
697
1436
|
}
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
scanned: flowResult.scanned,
|
|
706
|
-
resolved: flowResult.resolved,
|
|
707
|
-
missing: flowResult.missing,
|
|
708
|
-
ambiguous: flowResult.ambiguous,
|
|
709
|
-
});
|
|
1437
|
+
if (!opts.disableVlocityNameMatchResolve) {
|
|
1438
|
+
const tPhase = performance.now();
|
|
1439
|
+
try {
|
|
1440
|
+
const { resolveVlocityNameMatches } = await import("../parsers/vlocity/name-match-resolver.js");
|
|
1441
|
+
const nm = resolveVlocityNameMatches(graph, resolved.orgId, postCtx);
|
|
1442
|
+
if (nm.scanned > 0)
|
|
1443
|
+
logger.info("live-ingest: vlocity name-match resolver", { ...nm });
|
|
710
1444
|
}
|
|
1445
|
+
catch (e) {
|
|
1446
|
+
postFailures.push("vlocity-name-match-resolver");
|
|
1447
|
+
logger.warn(`vlocity-name-match-resolver:failed:${e.message}`);
|
|
1448
|
+
wedgeWarnings.push(`vlocity-name-match-resolver:failed:${e.message}`);
|
|
1449
|
+
}
|
|
1450
|
+
profiler.phase("resolver:vlocity-name-match", performance.now() - tPhase);
|
|
711
1451
|
}
|
|
712
|
-
|
|
713
|
-
|
|
1452
|
+
if (!opts.disableApexRefResolve) {
|
|
1453
|
+
const tPhase = performance.now();
|
|
1454
|
+
try {
|
|
1455
|
+
const { resolveApexReferences } = await import("../parsers/apex/ref-resolver.js");
|
|
1456
|
+
const { apexRefNoWorkResult } = await import("./resolver-gates.js");
|
|
1457
|
+
// Skip its whole-graph reads when it provably has no input and no
|
|
1458
|
+
// earlier output to prune (same result object it would return).
|
|
1459
|
+
const ar = apexRefNoWorkResult(graph, resolved.orgId) ??
|
|
1460
|
+
resolveApexReferences(graph, resolved.orgId, postCtx);
|
|
1461
|
+
if (ar.edgesEmitted > 0 || ar.pathsUnresolved > 0 || ar.dynamicSoqlFieldsSkipped > 0)
|
|
1462
|
+
logger.info("live-ingest: apex ref resolver", { ...ar });
|
|
1463
|
+
}
|
|
1464
|
+
catch (e) {
|
|
1465
|
+
postFailures.push("apex-ref-resolver");
|
|
1466
|
+
logger.warn(`apex-ref-resolver:failed:${e.message}`);
|
|
1467
|
+
wedgeWarnings.push(`apex-ref-resolver:failed:${e.message}`);
|
|
1468
|
+
}
|
|
1469
|
+
profiler.phase("resolver:apex-ref", performance.now() - tPhase);
|
|
714
1470
|
}
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
1471
|
+
// `ApexClass:X` references to an interface X (no class X): move them onto
|
|
1472
|
+
// `ApexInterface:X`. Before arity, which then sees interface method calls
|
|
1473
|
+
// under their real owner.
|
|
1474
|
+
let interfaceRefsRetargeted = 0;
|
|
1475
|
+
if (!opts.disableInterfaceRetarget) {
|
|
1476
|
+
const tPhase = performance.now();
|
|
1477
|
+
try {
|
|
1478
|
+
const names = deltaHooks?.retargetInterfaceNames();
|
|
1479
|
+
interfaceRefsRetargeted = retargetApexInterfaceRefs(graph, resolved.orgId, names ? { interfaceNames: names } : {});
|
|
1480
|
+
if (interfaceRefsRetargeted > 0) {
|
|
1481
|
+
logger.info("live-ingest: apex interface references retargeted", {
|
|
1482
|
+
interfaceRefsRetargeted,
|
|
1483
|
+
});
|
|
1484
|
+
}
|
|
1485
|
+
}
|
|
1486
|
+
catch (e) {
|
|
1487
|
+
postFailures.push("apex interface retarget");
|
|
1488
|
+
logger.warn("live-ingest: apex interface retarget failed", { err: e.message });
|
|
1489
|
+
}
|
|
1490
|
+
profiler.phase("resolver:interface-retarget", performance.now() - tPhase);
|
|
1491
|
+
}
|
|
1492
|
+
if (!opts.disableArityResolve) {
|
|
1493
|
+
const tPhase = performance.now();
|
|
1494
|
+
try {
|
|
1495
|
+
const scope = deltaHooks?.arityScope();
|
|
1496
|
+
const arityResult = resolveApexMethodArity(graph, {
|
|
1497
|
+
orgId: resolved.orgId,
|
|
1498
|
+
ctx: postCtx,
|
|
1499
|
+
...(scope ? { scope } : {}),
|
|
730
1500
|
});
|
|
1501
|
+
arityResolved = arityResult.resolved;
|
|
1502
|
+
if (arityResult.scanned > 0) {
|
|
1503
|
+
logger.info("live-ingest: apex method arity resolver", {
|
|
1504
|
+
scanned: arityResult.scanned,
|
|
1505
|
+
resolved: arityResult.resolved,
|
|
1506
|
+
ambiguous: arityResult.ambiguous,
|
|
1507
|
+
unresolved: arityResult.unresolved,
|
|
1508
|
+
edgesEmitted: arityResult.edgesEmitted,
|
|
1509
|
+
});
|
|
1510
|
+
}
|
|
1511
|
+
}
|
|
1512
|
+
catch (e) {
|
|
1513
|
+
postFailures.push("apex arity resolver");
|
|
1514
|
+
logger.warn("live-ingest: apex arity resolver failed", { err: e.message });
|
|
731
1515
|
}
|
|
1516
|
+
profiler.phase("resolver:arity", performance.now() - tPhase);
|
|
732
1517
|
}
|
|
733
|
-
|
|
734
|
-
|
|
1518
|
+
// A formula's second-hop field guess moves onto the CustomField that
|
|
1519
|
+
// declares that relationshipName.
|
|
1520
|
+
if (!opts.disableFormulaHopResolve) {
|
|
1521
|
+
const tPhase = performance.now();
|
|
1522
|
+
try {
|
|
1523
|
+
const sources = deltaHooks?.hopSources();
|
|
1524
|
+
const hop = resolveFormulaRelationshipHops(graph, resolved.orgId, sources ? { sources } : {});
|
|
1525
|
+
if (hop.moved > 0)
|
|
1526
|
+
logger.info("live-ingest: formula relationship hops resolved", { ...hop });
|
|
1527
|
+
}
|
|
1528
|
+
catch (e) {
|
|
1529
|
+
postFailures.push("formula hop resolver");
|
|
1530
|
+
logger.warn("live-ingest: formula hop resolver failed", { err: e.message });
|
|
1531
|
+
}
|
|
1532
|
+
profiler.phase("resolver:formula-hop", performance.now() - tPhase);
|
|
735
1533
|
}
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
1534
|
+
// Reflection walker: runs BEFORE the dangling-edge audit so any
|
|
1535
|
+
// REFERENCES edges it emits are validated by the audit alongside
|
|
1536
|
+
// parser-emitted edges. Default on; disable via disableReflectionWalker.
|
|
1537
|
+
if (!opts.disableReflectionWalker) {
|
|
1538
|
+
const tPhase = performance.now();
|
|
1539
|
+
try {
|
|
1540
|
+
const { walkBlobsForReferences } = await import("../parsers/generic/reflection-walker.js");
|
|
1541
|
+
const sourceQnames = deltaHooks?.reflectionSources();
|
|
1542
|
+
const reflectionResult = walkBlobsForReferences(graph, {
|
|
1543
|
+
orgId: resolved.orgId,
|
|
1544
|
+
ctx: postCtx,
|
|
1545
|
+
...(sourceQnames ? { sourceQnames } : {}),
|
|
1546
|
+
});
|
|
1547
|
+
reflectionEdges = reflectionResult.edgesEmitted;
|
|
1548
|
+
if (reflectionEdges > 0) {
|
|
1549
|
+
logger.info("live-ingest: reflection walker emitted REFERENCES edges", {
|
|
1550
|
+
scanned: reflectionResult.scanned,
|
|
1551
|
+
edges: reflectionEdges,
|
|
1552
|
+
truncated: reflectionResult.truncatedSources,
|
|
1553
|
+
ambiguous: reflectionResult.ambiguousMatches,
|
|
1554
|
+
});
|
|
1555
|
+
}
|
|
1556
|
+
}
|
|
1557
|
+
catch (e) {
|
|
1558
|
+
postFailures.push("reflection walker");
|
|
1559
|
+
logger.warn("live-ingest: reflection walker failed", {
|
|
1560
|
+
err: e.message,
|
|
754
1561
|
});
|
|
755
1562
|
}
|
|
1563
|
+
profiler.phase("resolver:reflection", performance.now() - tPhase);
|
|
756
1564
|
}
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
1565
|
+
// The audit only reports (it changes nothing): a delta skips it.
|
|
1566
|
+
if (!opts.disableAudit && !deltaHooks) {
|
|
1567
|
+
const tPhase = performance.now();
|
|
1568
|
+
try {
|
|
1569
|
+
const auditResult = auditDanglingEdges(graph, resolved.orgId, { sampleSize: 25 });
|
|
1570
|
+
danglingEdges = auditResult.danglingCount;
|
|
1571
|
+
if (auditResult.danglingCount > 0) {
|
|
1572
|
+
logger.info("live-ingest: graph audit found dangling edges", {
|
|
1573
|
+
totalEdges: auditResult.totalEdges,
|
|
1574
|
+
danglingCount: auditResult.danglingCount,
|
|
1575
|
+
platformRefs: auditResult.platformRefCount,
|
|
1576
|
+
unexpected: auditResult.unexpectedCount,
|
|
1577
|
+
byRel: auditResult.byRel,
|
|
1578
|
+
byDstPrefix: auditResult.byDstPrefix,
|
|
1579
|
+
});
|
|
1580
|
+
}
|
|
1581
|
+
}
|
|
1582
|
+
catch (e) {
|
|
1583
|
+
postFailures.push("graph audit");
|
|
1584
|
+
logger.warn("live-ingest: graph audit failed", { err: e.message });
|
|
1585
|
+
}
|
|
1586
|
+
profiler.phase("audit", performance.now() - tPhase);
|
|
761
1587
|
}
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
1588
|
+
const completedIso = new Date().toISOString();
|
|
1589
|
+
// N4: when incremental skipped changed members it couldn't fetch, do NOT
|
|
1590
|
+
// advance the watermark past them — hold it at the prior value so the next
|
|
1591
|
+
// incremental re-detects (and keeps warning about) those changes instead of
|
|
1592
|
+
// stepping over them forever. generation still increments / in-progress
|
|
1593
|
+
// still clears (markSyncComplete does both regardless of the timestamp).
|
|
1594
|
+
const holdWatermark = incrementalUnfetched > 0 && existing?.lastSyncedAt != null;
|
|
1595
|
+
const watermarkIso = holdWatermark
|
|
1596
|
+
? new Date(existing.lastSyncedAt).toISOString()
|
|
1597
|
+
: completedIso;
|
|
1598
|
+
// N-1: the held queue has not dispatched yet, so its lazy `onWork` flag
|
|
1599
|
+
// may not be set. Make `embeddings_pending` durable BEFORE the sync is
|
|
1600
|
+
// marked complete: a crash between here and the drain must leave the
|
|
1601
|
+
// next ingest knowing vectors are missing (it backfills them).
|
|
1602
|
+
if (embedQueue && !wasEmbeddingsPending && embedQueue.hasVectorWork()) {
|
|
1603
|
+
try {
|
|
1604
|
+
graph.setEmbeddingsPending?.(resolved.orgId, true);
|
|
1605
|
+
}
|
|
1606
|
+
catch (e) {
|
|
1607
|
+
logger.warn("live-ingest: setEmbeddingsPending failed", { err: e.message });
|
|
1608
|
+
}
|
|
1609
|
+
}
|
|
1610
|
+
const populate = async () => {
|
|
1611
|
+
if (!opts.analysisDb)
|
|
1612
|
+
return;
|
|
1613
|
+
const tDead = performance.now();
|
|
1614
|
+
try {
|
|
1615
|
+
await populateAnalysisTables(graph, resolved.orgId, opts.analysisDb);
|
|
1616
|
+
}
|
|
1617
|
+
catch (e) {
|
|
1618
|
+
postFailures.push("populate analysis tables");
|
|
1619
|
+
logger.warn("live-ingest: populate analysis tables failed", {
|
|
1620
|
+
err: e.message,
|
|
773
1621
|
});
|
|
774
1622
|
}
|
|
1623
|
+
profiler.phase("deadcode", performance.now() - tDead);
|
|
1624
|
+
};
|
|
1625
|
+
// T15 I-2: a delta whose resolver or dead-code pass failed is not a clean
|
|
1626
|
+
// sync. Populate runs before the sync is marked complete, and any failure
|
|
1627
|
+
// throws here: the pending marker and member rows stay as they were, so the
|
|
1628
|
+
// next delta replays the touched members with every resolver at full scope
|
|
1629
|
+
// (the outer catch marks the sync failed; the generation does not move).
|
|
1630
|
+
if (deltaHooks) {
|
|
1631
|
+
await populate();
|
|
1632
|
+
if (postFailures.length > 0) {
|
|
1633
|
+
throw new Error(`delta: post-merge step(s) failed (${postFailures.join(", ")}); not finalized, the next delta re-derives at full scope`);
|
|
1634
|
+
}
|
|
1635
|
+
}
|
|
1636
|
+
try {
|
|
1637
|
+
// W1.5-08: markSyncComplete subsumes the old touchSync — it updates
|
|
1638
|
+
// last_synced_at AND increments sync_generation AND clears the
|
|
1639
|
+
// in-progress flag, atomically in one transaction.
|
|
1640
|
+
graph.markSyncComplete(resolved.orgId, watermarkIso);
|
|
775
1641
|
}
|
|
776
1642
|
catch (e) {
|
|
777
|
-
logger.warn("live-ingest:
|
|
1643
|
+
logger.warn("live-ingest: markSyncComplete failed", { err: e.message });
|
|
778
1644
|
}
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
1645
|
+
opts.onPhase?.("sync-complete");
|
|
1646
|
+
if (!deltaHooks)
|
|
1647
|
+
await populate();
|
|
1648
|
+
// Task 11c: the ingest is complete; record what each member owns.
|
|
1649
|
+
if (deltaHooks)
|
|
1650
|
+
deltaHooks.finalize();
|
|
1651
|
+
else if (memberRows && !streamAborted) {
|
|
1652
|
+
try {
|
|
1653
|
+
graph.replaceMemberRows?.(resolved.orgId, [...memberRows.values()], memberRowsGeneration());
|
|
1654
|
+
}
|
|
1655
|
+
catch (e) {
|
|
1656
|
+
logger.warn("live-ingest: member rows write failed", { err: e.message });
|
|
1657
|
+
}
|
|
1658
|
+
}
|
|
1659
|
+
// T15 N5-1: surface members skipped after repeated parse-worker failures.
|
|
1660
|
+
const parseQuarantined = deltaHooks
|
|
1661
|
+
? (deltaHooks.quarantinedCount?.() ?? 0)
|
|
1662
|
+
: memberRows
|
|
1663
|
+
? [...memberRows.values()].filter((r) => isQuarantinedRow(r)).length
|
|
1664
|
+
: 0;
|
|
1665
|
+
if (parseQuarantined > 0) {
|
|
1666
|
+
logger.warn("live-ingest: members skipped after repeated parse-worker failures (until their content changes)", { count: parseQuarantined });
|
|
1667
|
+
}
|
|
1668
|
+
// Graph ready: merges, resolvers and the dead-code cache are done. Only
|
|
1669
|
+
// embeddings may still be in flight.
|
|
1670
|
+
const graphReadyMs = Date.now() - startedAt;
|
|
1671
|
+
console.log(`graph ready in ${(graphReadyMs / 1000).toFixed(1)}s`);
|
|
1672
|
+
profiler.phase("graph-ready", graphReadyMs, true);
|
|
1673
|
+
let embeddings;
|
|
1674
|
+
if (embedQueue) {
|
|
1675
|
+
const q = embedQueue;
|
|
1676
|
+
let pendingLeft = 0;
|
|
1677
|
+
if (opts.waitEmbeddings === false) {
|
|
1678
|
+
// Cache hits are cheap: write them before returning, so a warm no-wait
|
|
1679
|
+
// run still completes its vectors. Only model work is deferred.
|
|
1680
|
+
try {
|
|
1681
|
+
q.flushCached();
|
|
1682
|
+
}
|
|
1683
|
+
catch (e) {
|
|
1684
|
+
// Items may have been dropped mid-flush: keep the flag set.
|
|
1685
|
+
if (!embedError)
|
|
1686
|
+
embedError = e;
|
|
1687
|
+
logger.warn("live-ingest: cached vector flush failed", { err: e.message });
|
|
1688
|
+
}
|
|
1689
|
+
pendingLeft = q.abandon() + q.stats.failed;
|
|
1690
|
+
const err = embedError;
|
|
1691
|
+
if (pendingLeft > 0 || err) {
|
|
1692
|
+
console.log(`embeddings deferred: ${pendingLeft} texts pending (cached vectors written: ${q.stats.cached}); find_similar uses text search until an ingest that waits for embeddings fills them`);
|
|
1693
|
+
}
|
|
1694
|
+
else {
|
|
1695
|
+
graph.setEmbeddingsPending?.(resolved.orgId, false);
|
|
1696
|
+
}
|
|
1697
|
+
}
|
|
1698
|
+
else {
|
|
1699
|
+
try {
|
|
1700
|
+
// drain() releases the held queue.
|
|
1701
|
+
await q.drain();
|
|
1702
|
+
}
|
|
1703
|
+
catch (e) {
|
|
1704
|
+
if (!embedError)
|
|
1705
|
+
embedError = e;
|
|
1706
|
+
}
|
|
1707
|
+
const doneMs = Date.now() - startedAt;
|
|
1708
|
+
const err = embedError;
|
|
1709
|
+
if (err) {
|
|
1710
|
+
pendingLeft = q.stats.failed;
|
|
1711
|
+
console.error(`embeddings failed: ${err.message} — ${q.stats.failed} texts left unembedded; they will be filled on the next ingest`);
|
|
1712
|
+
}
|
|
1713
|
+
else {
|
|
1714
|
+
graph.setEmbeddingsPending?.(resolved.orgId, false);
|
|
1715
|
+
console.log(`embeddings done in ${(doneMs / 1000).toFixed(1)}s (cached ${q.stats.cached} / computed ${q.stats.computed})`);
|
|
1716
|
+
}
|
|
1717
|
+
profiler.phase("embeddings", doneMs, true);
|
|
1718
|
+
}
|
|
1719
|
+
const err = embedError;
|
|
1720
|
+
embeddings = {
|
|
1721
|
+
...q.stats,
|
|
1722
|
+
pending: pendingLeft,
|
|
1723
|
+
backfilled,
|
|
1724
|
+
doneMs: Date.now() - startedAt,
|
|
1725
|
+
...(err ? { error: err.message } : {}),
|
|
1726
|
+
};
|
|
1727
|
+
}
|
|
1728
|
+
if (opts.snapshotStore) {
|
|
1729
|
+
try {
|
|
1730
|
+
opts.snapshotStore.prune(resolved.orgId, opts.snapshotRetentionDays ?? 30);
|
|
1731
|
+
}
|
|
1732
|
+
catch (e) {
|
|
1733
|
+
logger.warn("live-ingest: snapshot prune failed", { err: e.message });
|
|
1734
|
+
}
|
|
1735
|
+
}
|
|
1736
|
+
return {
|
|
1737
|
+
orgId: resolved.orgId,
|
|
1738
|
+
capabilities: caps,
|
|
1739
|
+
mode,
|
|
1740
|
+
membersProcessed,
|
|
1741
|
+
parseErrors,
|
|
1742
|
+
deletions,
|
|
1743
|
+
durationMs: Date.now() - startedAt,
|
|
1744
|
+
crossFlavorEdges,
|
|
1745
|
+
arityResolved,
|
|
1746
|
+
parseQuarantined,
|
|
1747
|
+
interfaceRefsRetargeted,
|
|
1748
|
+
flowMethodsResolved,
|
|
1749
|
+
vlocityApex: vlocityApexResult,
|
|
1750
|
+
drRelationships: drRelationshipsResult,
|
|
1751
|
+
danglingEdges,
|
|
1752
|
+
reflectionEdges,
|
|
1753
|
+
overlap: overlapResult,
|
|
1754
|
+
warnings: [
|
|
1755
|
+
...skipReport.skips.map((s) => `${s.label}: ${s.reason}`),
|
|
1756
|
+
// Phase 1.5 W1.5-02 namespaced wedge strings. Format documented in
|
|
1757
|
+
// PLAN.md: `wedge:<source>:<stage>:<detail>`. Forward-compatible
|
|
1758
|
+
// with Phase 1's structured-warnings refactor (W1-01).
|
|
1759
|
+
...wedgeWarnings,
|
|
1760
|
+
],
|
|
1761
|
+
graphReadyMs,
|
|
1762
|
+
...(streamAborted ? { streamAborted: true } : {}),
|
|
1763
|
+
...(embeddings ? { embeddings } : {}),
|
|
1764
|
+
};
|
|
783
1765
|
}
|
|
784
1766
|
catch (e) {
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
1767
|
+
embedQueue?.abandon();
|
|
1768
|
+
// Never leave a bulk window (open transaction, dropped indexes) or a
|
|
1769
|
+
// batch window (open transaction) behind.
|
|
788
1770
|
try {
|
|
789
|
-
|
|
1771
|
+
graph.endBatch?.();
|
|
790
1772
|
}
|
|
791
|
-
catch
|
|
792
|
-
|
|
793
|
-
|
|
1773
|
+
catch {
|
|
1774
|
+
/* best-effort */
|
|
1775
|
+
}
|
|
1776
|
+
try {
|
|
1777
|
+
graph.endBulk?.();
|
|
1778
|
+
}
|
|
1779
|
+
catch {
|
|
1780
|
+
/* best-effort */
|
|
1781
|
+
}
|
|
1782
|
+
// W1.5-08: ingest threw before reaching markSyncComplete. Flip
|
|
1783
|
+
// sync_in_progress back to 0 (and clear sync_started_at) so concurrent
|
|
1784
|
+
// MCP readers don't see an orphaned in_progress flag forever. Generation
|
|
1785
|
+
// is intentionally NOT incremented — failed ingests don't count as
|
|
1786
|
+
// fresh graph data. Re-throw so callers (CLI / MCP job runner) see the
|
|
1787
|
+
// original error unchanged.
|
|
1788
|
+
try {
|
|
1789
|
+
graph.markSyncFailed(resolved.orgId);
|
|
1790
|
+
}
|
|
1791
|
+
catch (markErr) {
|
|
1792
|
+
logger.warn("live-ingest: markSyncFailed failed", {
|
|
1793
|
+
err: markErr.message,
|
|
794
1794
|
});
|
|
795
1795
|
}
|
|
1796
|
+
throw e;
|
|
796
1797
|
}
|
|
797
|
-
|
|
1798
|
+
finally {
|
|
1799
|
+
profiler.close();
|
|
798
1800
|
try {
|
|
799
|
-
|
|
1801
|
+
await ownedEmbedder?.close();
|
|
800
1802
|
}
|
|
801
|
-
catch
|
|
802
|
-
|
|
1803
|
+
catch {
|
|
1804
|
+
/* best-effort */
|
|
1805
|
+
}
|
|
1806
|
+
try {
|
|
1807
|
+
embedCache?.close();
|
|
1808
|
+
}
|
|
1809
|
+
catch {
|
|
1810
|
+
/* best-effort */
|
|
803
1811
|
}
|
|
804
1812
|
}
|
|
805
|
-
return {
|
|
806
|
-
orgId: resolved.orgId,
|
|
807
|
-
capabilities: caps,
|
|
808
|
-
mode,
|
|
809
|
-
membersProcessed,
|
|
810
|
-
parseErrors,
|
|
811
|
-
deletions,
|
|
812
|
-
durationMs: Date.now() - startedAt,
|
|
813
|
-
crossFlavorEdges,
|
|
814
|
-
arityResolved,
|
|
815
|
-
flowMethodsResolved,
|
|
816
|
-
danglingEdges,
|
|
817
|
-
reflectionEdges,
|
|
818
|
-
overlap: overlapResult,
|
|
819
|
-
warnings: skipReport.skips.map((s) => `${s.label}: ${s.reason}`),
|
|
820
|
-
};
|
|
821
1813
|
}
|
|
822
1814
|
//# sourceMappingURL=live-ingest.js.map
|