@ryanstark24/sfgraph-core 1.1.8 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/analyze/__tests__/audit-graph.test.js +79 -0
- package/dist/analyze/__tests__/audit-graph.test.js.map +1 -1
- package/dist/analyze/__tests__/edge-filters.test.d.ts +2 -0
- package/dist/analyze/__tests__/edge-filters.test.d.ts.map +1 -0
- package/dist/analyze/__tests__/edge-filters.test.js +66 -0
- package/dist/analyze/__tests__/edge-filters.test.js.map +1 -0
- package/dist/analyze/__tests__/find-nodes.test.d.ts +2 -0
- package/dist/analyze/__tests__/find-nodes.test.d.ts.map +1 -0
- package/dist/analyze/__tests__/find-nodes.test.js +129 -0
- package/dist/analyze/__tests__/find-nodes.test.js.map +1 -0
- package/dist/analyze/__tests__/findings.test.d.ts +2 -0
- package/dist/analyze/__tests__/findings.test.d.ts.map +1 -0
- package/dist/analyze/__tests__/findings.test.js +100 -0
- package/dist/analyze/__tests__/findings.test.js.map +1 -0
- package/dist/analyze/__tests__/governor-risks-detection.test.js +90 -0
- package/dist/analyze/__tests__/governor-risks-detection.test.js.map +1 -1
- package/dist/analyze/__tests__/security-fls-filter.test.d.ts +2 -0
- package/dist/analyze/__tests__/security-fls-filter.test.d.ts.map +1 -0
- package/dist/analyze/__tests__/security-fls-filter.test.js +23 -0
- package/dist/analyze/__tests__/security-fls-filter.test.js.map +1 -0
- package/dist/analyze/audit-graph.d.ts +26 -2
- package/dist/analyze/audit-graph.d.ts.map +1 -1
- package/dist/analyze/audit-graph.js +52 -5
- package/dist/analyze/audit-graph.js.map +1 -1
- package/dist/analyze/dependencies.d.ts +2 -2
- package/dist/analyze/dependencies.d.ts.map +1 -1
- package/dist/analyze/dependencies.js +9 -3
- package/dist/analyze/dependencies.js.map +1 -1
- package/dist/analyze/dependents.d.ts +6 -1
- package/dist/analyze/dependents.d.ts.map +1 -1
- package/dist/analyze/dependents.js +12 -2
- package/dist/analyze/dependents.js.map +1 -1
- package/dist/analyze/edge-filters.d.ts +37 -0
- package/dist/analyze/edge-filters.d.ts.map +1 -0
- package/dist/analyze/edge-filters.js +51 -0
- package/dist/analyze/edge-filters.js.map +1 -0
- package/dist/analyze/find-nodes.d.ts +38 -0
- package/dist/analyze/find-nodes.d.ts.map +1 -0
- package/dist/analyze/find-nodes.js +51 -0
- package/dist/analyze/find-nodes.js.map +1 -0
- package/dist/analyze/findings.d.ts +86 -0
- package/dist/analyze/findings.d.ts.map +1 -0
- package/dist/analyze/findings.js +141 -0
- package/dist/analyze/findings.js.map +1 -0
- package/dist/analyze/index.d.ts +2 -0
- package/dist/analyze/index.d.ts.map +1 -1
- package/dist/analyze/index.js +2 -0
- package/dist/analyze/index.js.map +1 -1
- package/dist/analyze/populate.d.ts.map +1 -1
- package/dist/analyze/populate.js +55 -10
- package/dist/analyze/populate.js.map +1 -1
- package/dist/analyze/security.d.ts +15 -0
- package/dist/analyze/security.d.ts.map +1 -1
- package/dist/analyze/security.js +59 -3
- package/dist/analyze/security.js.map +1 -1
- package/dist/analyze/wip-impact.d.ts.map +1 -1
- package/dist/analyze/wip-impact.js +12 -2
- package/dist/analyze/wip-impact.js.map +1 -1
- package/dist/domain/edge-fact.d.ts +25 -0
- package/dist/domain/edge-fact.d.ts.map +1 -1
- package/dist/domain/metadata-category.d.ts +1 -0
- package/dist/domain/metadata-category.d.ts.map +1 -1
- package/dist/domain/metadata-category.js +1 -0
- package/dist/domain/metadata-category.js.map +1 -1
- package/dist/domain/rel-types.d.ts +4 -0
- package/dist/domain/rel-types.d.ts.map +1 -1
- package/dist/domain/rel-types.js +7 -0
- package/dist/domain/rel-types.js.map +1 -1
- package/dist/embedding/__tests__/embed.test.d.ts +2 -0
- package/dist/embedding/__tests__/embed.test.d.ts.map +1 -0
- package/dist/embedding/__tests__/embed.test.js +72 -0
- package/dist/embedding/__tests__/embed.test.js.map +1 -0
- package/dist/embedding/embed.d.ts.map +1 -1
- package/dist/embedding/embed.js +14 -1
- package/dist/embedding/embed.js.map +1 -1
- package/dist/extractors/interfaces/metadata-source.d.ts +11 -0
- package/dist/extractors/interfaces/metadata-source.d.ts.map +1 -1
- package/dist/extractors/live-org/__tests__/bulk-retrieve.test.js.map +1 -1
- package/dist/extractors/live-org/__tests__/custom-settings.test.d.ts +2 -0
- package/dist/extractors/live-org/__tests__/custom-settings.test.d.ts.map +1 -0
- package/dist/extractors/live-org/__tests__/custom-settings.test.js +79 -0
- package/dist/extractors/live-org/__tests__/custom-settings.test.js.map +1 -0
- package/dist/extractors/live-org/__tests__/extractor-enrichment.test.js +70 -8
- package/dist/extractors/live-org/__tests__/extractor-enrichment.test.js.map +1 -1
- package/dist/extractors/live-org/__tests__/liveness.test.d.ts +2 -0
- package/dist/extractors/live-org/__tests__/liveness.test.d.ts.map +1 -0
- package/dist/extractors/live-org/__tests__/liveness.test.js +148 -0
- package/dist/extractors/live-org/__tests__/liveness.test.js.map +1 -0
- package/dist/extractors/live-org/__tests__/mcd-baseline.test.d.ts +2 -0
- package/dist/extractors/live-org/__tests__/mcd-baseline.test.d.ts.map +1 -0
- package/dist/extractors/live-org/__tests__/mcd-baseline.test.js +177 -0
- package/dist/extractors/live-org/__tests__/mcd-baseline.test.js.map +1 -0
- package/dist/extractors/live-org/__tests__/metadata-read-batch.test.d.ts +2 -0
- package/dist/extractors/live-org/__tests__/metadata-read-batch.test.d.ts.map +1 -0
- package/dist/extractors/live-org/__tests__/metadata-read-batch.test.js +75 -0
- package/dist/extractors/live-org/__tests__/metadata-read-batch.test.js.map +1 -0
- package/dist/extractors/live-org/__tests__/omnistudio-retrieve.test.d.ts +2 -0
- package/dist/extractors/live-org/__tests__/omnistudio-retrieve.test.d.ts.map +1 -0
- package/dist/extractors/live-org/__tests__/omnistudio-retrieve.test.js +135 -0
- package/dist/extractors/live-org/__tests__/omnistudio-retrieve.test.js.map +1 -0
- package/dist/extractors/live-org/__tests__/pools.test.js +4 -1
- package/dist/extractors/live-org/__tests__/pools.test.js.map +1 -1
- package/dist/extractors/live-org/__tests__/rate-limit.test.js +1 -6
- package/dist/extractors/live-org/__tests__/rate-limit.test.js.map +1 -1
- package/dist/extractors/live-org/__tests__/soql-rebatch.test.d.ts +2 -0
- package/dist/extractors/live-org/__tests__/soql-rebatch.test.d.ts.map +1 -0
- package/dist/extractors/live-org/__tests__/soql-rebatch.test.js +118 -0
- package/dist/extractors/live-org/__tests__/soql-rebatch.test.js.map +1 -0
- package/dist/extractors/live-org/__tests__/stop-waiting.test.d.ts +2 -0
- package/dist/extractors/live-org/__tests__/stop-waiting.test.d.ts.map +1 -0
- package/dist/extractors/live-org/__tests__/stop-waiting.test.js +243 -0
- package/dist/extractors/live-org/__tests__/stop-waiting.test.js.map +1 -0
- package/dist/extractors/live-org/__tests__/vlocity-runner.test.js +27 -0
- package/dist/extractors/live-org/__tests__/vlocity-runner.test.js.map +1 -1
- package/dist/extractors/live-org/__tests__/watchdog-budgets.test.d.ts +2 -0
- package/dist/extractors/live-org/__tests__/watchdog-budgets.test.d.ts.map +1 -0
- package/dist/extractors/live-org/__tests__/watchdog-budgets.test.js +40 -0
- package/dist/extractors/live-org/__tests__/watchdog-budgets.test.js.map +1 -0
- package/dist/extractors/live-org/__tests__/wedge-isolation.test.d.ts +2 -0
- package/dist/extractors/live-org/__tests__/wedge-isolation.test.d.ts.map +1 -0
- package/dist/extractors/live-org/__tests__/wedge-isolation.test.js +270 -0
- package/dist/extractors/live-org/__tests__/wedge-isolation.test.js.map +1 -0
- package/dist/extractors/live-org/bulk-retrieve.d.ts +176 -0
- package/dist/extractors/live-org/bulk-retrieve.d.ts.map +1 -1
- package/dist/extractors/live-org/bulk-retrieve.js +509 -71
- package/dist/extractors/live-org/bulk-retrieve.js.map +1 -1
- package/dist/extractors/live-org/capabilities.d.ts.map +1 -1
- package/dist/extractors/live-org/capabilities.js.map +1 -1
- package/dist/extractors/live-org/extractors/__tests__/apex.test.d.ts +2 -0
- package/dist/extractors/live-org/extractors/__tests__/apex.test.d.ts.map +1 -0
- package/dist/extractors/live-org/extractors/__tests__/apex.test.js +73 -0
- package/dist/extractors/live-org/extractors/__tests__/apex.test.js.map +1 -0
- package/dist/extractors/live-org/extractors/__tests__/lwc.parallel.test.d.ts +2 -0
- package/dist/extractors/live-org/extractors/__tests__/lwc.parallel.test.d.ts.map +1 -0
- package/dist/extractors/live-org/extractors/__tests__/lwc.parallel.test.js +183 -0
- package/dist/extractors/live-org/extractors/__tests__/lwc.parallel.test.js.map +1 -0
- package/dist/extractors/live-org/extractors/apex.d.ts.map +1 -1
- package/dist/extractors/live-org/extractors/apex.js +76 -51
- package/dist/extractors/live-org/extractors/apex.js.map +1 -1
- package/dist/extractors/live-org/extractors/custom-settings.d.ts +3 -0
- package/dist/extractors/live-org/extractors/custom-settings.d.ts.map +1 -0
- package/dist/extractors/live-org/extractors/custom-settings.js +66 -0
- package/dist/extractors/live-org/extractors/custom-settings.js.map +1 -0
- package/dist/extractors/live-org/extractors/flow.d.ts.map +1 -1
- package/dist/extractors/live-org/extractors/flow.js.map +1 -1
- package/dist/extractors/live-org/extractors/generic-metadata.d.ts.map +1 -1
- package/dist/extractors/live-org/extractors/generic-metadata.js.map +1 -1
- package/dist/extractors/live-org/extractors/generic-sobject.js.map +1 -1
- package/dist/extractors/live-org/extractors/lwc.d.ts.map +1 -1
- package/dist/extractors/live-org/extractors/lwc.js +81 -41
- package/dist/extractors/live-org/extractors/lwc.js.map +1 -1
- package/dist/extractors/live-org/extractors/mcd-baseline.d.ts +60 -0
- package/dist/extractors/live-org/extractors/mcd-baseline.d.ts.map +1 -0
- package/dist/extractors/live-org/extractors/mcd-baseline.js +118 -0
- package/dist/extractors/live-org/extractors/mcd-baseline.js.map +1 -0
- package/dist/extractors/live-org/extractors/object.d.ts.map +1 -1
- package/dist/extractors/live-org/extractors/object.js.map +1 -1
- package/dist/extractors/live-org/extractors/omnistudio-retrieve.d.ts +23 -0
- package/dist/extractors/live-org/extractors/omnistudio-retrieve.d.ts.map +1 -0
- package/dist/extractors/live-org/extractors/omnistudio-retrieve.js +172 -0
- package/dist/extractors/live-org/extractors/omnistudio-retrieve.js.map +1 -0
- package/dist/extractors/live-org/extractors/omnistudio.d.ts +8 -1
- package/dist/extractors/live-org/extractors/omnistudio.d.ts.map +1 -1
- package/dist/extractors/live-org/extractors/omnistudio.js +59 -6
- package/dist/extractors/live-org/extractors/omnistudio.js.map +1 -1
- package/dist/extractors/live-org/rate-limit.d.ts +98 -15
- package/dist/extractors/live-org/rate-limit.d.ts.map +1 -1
- package/dist/extractors/live-org/rate-limit.js +278 -41
- package/dist/extractors/live-org/rate-limit.js.map +1 -1
- package/dist/extractors/live-org/vlocity/runner.d.ts +1 -1
- package/dist/extractors/live-org/vlocity/runner.d.ts.map +1 -1
- package/dist/extractors/live-org/vlocity/runner.js +49 -14
- package/dist/extractors/live-org/vlocity/runner.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/ingest/__tests__/config-values.test.d.ts +2 -0
- package/dist/ingest/__tests__/config-values.test.d.ts.map +1 -0
- package/dist/ingest/__tests__/config-values.test.js +65 -0
- package/dist/ingest/__tests__/config-values.test.js.map +1 -0
- package/dist/ingest/__tests__/full-sync-deletion.test.js +11 -4
- package/dist/ingest/__tests__/full-sync-deletion.test.js.map +1 -1
- package/dist/ingest/__tests__/live-ingest.detect-deletions-guard.test.d.ts +10 -0
- package/dist/ingest/__tests__/live-ingest.detect-deletions-guard.test.d.ts.map +1 -0
- package/dist/ingest/__tests__/live-ingest.detect-deletions-guard.test.js +251 -0
- package/dist/ingest/__tests__/live-ingest.detect-deletions-guard.test.js.map +1 -0
- package/dist/ingest/__tests__/live-ingest.test.js.map +1 -1
- package/dist/ingest/__tests__/multi-org-orchestrator.test.js +3 -0
- package/dist/ingest/__tests__/multi-org-orchestrator.test.js.map +1 -1
- package/dist/ingest/__tests__/parser-dispatch.test.d.ts +2 -0
- package/dist/ingest/__tests__/parser-dispatch.test.d.ts.map +1 -0
- package/dist/ingest/__tests__/parser-dispatch.test.js +55 -0
- package/dist/ingest/__tests__/parser-dispatch.test.js.map +1 -0
- package/dist/ingest/__tests__/vlocity-dispatch-integration.test.d.ts +2 -0
- package/dist/ingest/__tests__/vlocity-dispatch-integration.test.d.ts.map +1 -0
- package/dist/ingest/__tests__/vlocity-dispatch-integration.test.js +134 -0
- package/dist/ingest/__tests__/vlocity-dispatch-integration.test.js.map +1 -0
- package/dist/ingest/detect-deletions-guard.d.ts +45 -0
- package/dist/ingest/detect-deletions-guard.d.ts.map +1 -0
- package/dist/ingest/detect-deletions-guard.js +63 -0
- package/dist/ingest/detect-deletions-guard.js.map +1 -0
- package/dist/ingest/live-ingest.d.ts +86 -0
- package/dist/ingest/live-ingest.d.ts.map +1 -1
- package/dist/ingest/live-ingest.js +783 -449
- package/dist/ingest/live-ingest.js.map +1 -1
- package/dist/parsers/__tests__/datatransform-structured.test.d.ts +2 -0
- package/dist/parsers/__tests__/datatransform-structured.test.d.ts.map +1 -0
- package/dist/parsers/__tests__/datatransform-structured.test.js +147 -0
- package/dist/parsers/__tests__/datatransform-structured.test.js.map +1 -0
- package/dist/parsers/__tests__/edge-provenance.test.d.ts +2 -0
- package/dist/parsers/__tests__/edge-provenance.test.d.ts.map +1 -0
- package/dist/parsers/__tests__/edge-provenance.test.js +47 -0
- package/dist/parsers/__tests__/edge-provenance.test.js.map +1 -0
- package/dist/parsers/__tests__/lwc-bindings.test.js +78 -3
- package/dist/parsers/__tests__/lwc-bindings.test.js.map +1 -1
- package/dist/parsers/__tests__/mcd-gap-fills.test.d.ts +2 -0
- package/dist/parsers/__tests__/mcd-gap-fills.test.d.ts.map +1 -0
- package/dist/parsers/__tests__/mcd-gap-fills.test.js +75 -0
- package/dist/parsers/__tests__/mcd-gap-fills.test.js.map +1 -0
- package/dist/parsers/__tests__/object-inline-fields.test.js +2 -2
- package/dist/parsers/__tests__/object-inline-fields.test.js.map +1 -1
- package/dist/parsers/__tests__/omnistudio-overlap.test.d.ts +2 -0
- package/dist/parsers/__tests__/omnistudio-overlap.test.d.ts.map +1 -0
- package/dist/parsers/__tests__/omnistudio-overlap.test.js +163 -0
- package/dist/parsers/__tests__/omnistudio-overlap.test.js.map +1 -0
- package/dist/parsers/__tests__/omnistudio.golden.test.js +5 -6
- package/dist/parsers/__tests__/omnistudio.golden.test.js.map +1 -1
- package/dist/parsers/__tests__/reflection-walker.test.d.ts +2 -0
- package/dist/parsers/__tests__/reflection-walker.test.d.ts.map +1 -0
- package/dist/parsers/__tests__/reflection-walker.test.js +162 -0
- package/dist/parsers/__tests__/reflection-walker.test.js.map +1 -0
- package/dist/parsers/__tests__/vlocity.golden.test.js +9 -13
- package/dist/parsers/__tests__/vlocity.golden.test.js.map +1 -1
- package/dist/parsers/apex/__tests__/ast-extractor.test.js +64 -4
- package/dist/parsers/apex/__tests__/ast-extractor.test.js.map +1 -1
- package/dist/parsers/apex/__tests__/empty-body.test.d.ts +2 -0
- package/dist/parsers/apex/__tests__/empty-body.test.d.ts.map +1 -0
- package/dist/parsers/apex/__tests__/empty-body.test.js +40 -0
- package/dist/parsers/apex/__tests__/empty-body.test.js.map +1 -0
- package/dist/parsers/apex/__tests__/is-test-attribute.test.d.ts +2 -0
- package/dist/parsers/apex/__tests__/is-test-attribute.test.d.ts.map +1 -0
- package/dist/parsers/apex/__tests__/is-test-attribute.test.js +56 -0
- package/dist/parsers/apex/__tests__/is-test-attribute.test.js.map +1 -0
- package/dist/parsers/apex/__tests__/regex-arg-count.test.d.ts +2 -0
- package/dist/parsers/apex/__tests__/regex-arg-count.test.d.ts.map +1 -0
- package/dist/parsers/apex/__tests__/regex-arg-count.test.js +82 -0
- package/dist/parsers/apex/__tests__/regex-arg-count.test.js.map +1 -0
- package/dist/parsers/apex/__tests__/regex-inloop.test.d.ts +2 -0
- package/dist/parsers/apex/__tests__/regex-inloop.test.d.ts.map +1 -0
- package/dist/parsers/apex/__tests__/regex-inloop.test.js +70 -0
- package/dist/parsers/apex/__tests__/regex-inloop.test.js.map +1 -0
- package/dist/parsers/apex/arity-resolver.d.ts.map +1 -1
- package/dist/parsers/apex/arity-resolver.js.map +1 -1
- package/dist/parsers/apex/ast-extractor.d.ts.map +1 -1
- package/dist/parsers/apex/ast-extractor.js +33 -1
- package/dist/parsers/apex/ast-extractor.js.map +1 -1
- package/dist/parsers/apex/class.d.ts.map +1 -1
- package/dist/parsers/apex/class.js +162 -5
- package/dist/parsers/apex/class.js.map +1 -1
- package/dist/parsers/apex/trigger.d.ts.map +1 -1
- package/dist/parsers/apex/trigger.js +10 -0
- package/dist/parsers/apex/trigger.js.map +1 -1
- package/dist/parsers/common.d.ts +4 -1
- package/dist/parsers/common.d.ts.map +1 -1
- package/dist/parsers/common.js +15 -2
- package/dist/parsers/common.js.map +1 -1
- package/dist/parsers/flow/__tests__/invocable-resolver.test.js.map +1 -1
- package/dist/parsers/flow/invocable-resolver.d.ts.map +1 -1
- package/dist/parsers/flow/invocable-resolver.js.map +1 -1
- package/dist/parsers/generic/reflection-walker.d.ts +90 -0
- package/dist/parsers/generic/reflection-walker.d.ts.map +1 -0
- package/dist/parsers/generic/reflection-walker.js +193 -0
- package/dist/parsers/generic/reflection-walker.js.map +1 -0
- package/dist/parsers/lwc/html-visitor.d.ts.map +1 -1
- package/dist/parsers/lwc/html-visitor.js +82 -3
- package/dist/parsers/lwc/html-visitor.js.map +1 -1
- package/dist/parsers/lwc/js-visitor.d.ts.map +1 -1
- package/dist/parsers/lwc/js-visitor.js +6 -2
- package/dist/parsers/lwc/js-visitor.js.map +1 -1
- package/dist/parsers/object/object.d.ts.map +1 -1
- package/dist/parsers/object/object.js +35 -5
- package/dist/parsers/object/object.js.map +1 -1
- package/dist/parsers/omnistudio/data-transform.d.ts.map +1 -1
- package/dist/parsers/omnistudio/data-transform.js +39 -6
- package/dist/parsers/omnistudio/data-transform.js.map +1 -1
- package/dist/parsers/omnistudio/integration-procedure.d.ts.map +1 -1
- package/dist/parsers/omnistudio/integration-procedure.js +3 -2
- package/dist/parsers/omnistudio/integration-procedure.js.map +1 -1
- package/dist/parsers/omnistudio/overlap-detector.d.ts +50 -0
- package/dist/parsers/omnistudio/overlap-detector.d.ts.map +1 -0
- package/dist/parsers/omnistudio/overlap-detector.js +122 -0
- package/dist/parsers/omnistudio/overlap-detector.js.map +1 -0
- package/dist/parsers/omnistudio/process.d.ts.map +1 -1
- package/dist/parsers/omnistudio/process.js +3 -2
- package/dist/parsers/omnistudio/process.js.map +1 -1
- package/dist/parsers/omnistudio/ui-card.d.ts.map +1 -1
- package/dist/parsers/omnistudio/ui-card.js +3 -2
- package/dist/parsers/omnistudio/ui-card.js.map +1 -1
- package/dist/parsers/rules/custom-metadata-type.yml +7 -0
- package/dist/parsers/rules/custom-setting.yml +24 -0
- package/dist/parsers/vlocity/common.d.ts +16 -0
- package/dist/parsers/vlocity/common.d.ts.map +1 -1
- package/dist/parsers/vlocity/common.js +62 -0
- package/dist/parsers/vlocity/common.js.map +1 -1
- package/dist/parsers/vlocity/data-raptor.d.ts.map +1 -1
- package/dist/parsers/vlocity/data-raptor.js +43 -14
- package/dist/parsers/vlocity/data-raptor.js.map +1 -1
- package/dist/render/__tests__/sarif.test.d.ts +2 -0
- package/dist/render/__tests__/sarif.test.d.ts.map +1 -0
- package/dist/render/__tests__/sarif.test.js +126 -0
- package/dist/render/__tests__/sarif.test.js.map +1 -0
- package/dist/render/sarif.d.ts +94 -0
- package/dist/render/sarif.d.ts.map +1 -0
- package/dist/render/sarif.js +131 -0
- package/dist/render/sarif.js.map +1 -0
- package/dist/storage/__tests__/analysis-tables-migration.test.js +1 -1
- package/dist/storage/__tests__/graph-store.test.js +13 -0
- package/dist/storage/__tests__/graph-store.test.js.map +1 -1
- package/dist/storage/__tests__/migrations.test.js +4 -4
- package/dist/storage/__tests__/rename-stability.test.d.ts +2 -0
- package/dist/storage/__tests__/rename-stability.test.d.ts.map +1 -0
- package/dist/storage/__tests__/rename-stability.test.js +135 -0
- package/dist/storage/__tests__/rename-stability.test.js.map +1 -0
- package/dist/storage/__tests__/sync-generation.test.d.ts +2 -0
- package/dist/storage/__tests__/sync-generation.test.d.ts.map +1 -0
- package/dist/storage/__tests__/sync-generation.test.js +159 -0
- package/dist/storage/__tests__/sync-generation.test.js.map +1 -0
- package/dist/storage/index.d.ts +1 -0
- package/dist/storage/index.d.ts.map +1 -1
- package/dist/storage/index.js +1 -0
- package/dist/storage/index.js.map +1 -1
- package/dist/storage/interfaces.d.ts +60 -0
- package/dist/storage/interfaces.d.ts.map +1 -1
- package/dist/storage/sqlite/graph-store.d.ts +21 -1
- package/dist/storage/sqlite/graph-store.d.ts.map +1 -1
- package/dist/storage/sqlite/graph-store.js +86 -0
- package/dist/storage/sqlite/graph-store.js.map +1 -1
- package/dist/storage/sqlite/migrations.d.ts.map +1 -1
- package/dist/storage/sqlite/migrations.js +38 -0
- package/dist/storage/sqlite/migrations.js.map +1 -1
- package/dist/storage/sqlite/rename-stability.d.ts +94 -0
- package/dist/storage/sqlite/rename-stability.d.ts.map +1 -0
- package/dist/storage/sqlite/rename-stability.js +92 -0
- package/dist/storage/sqlite/rename-stability.js.map +1 -0
- package/package.json +4 -1
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
import { describe, expect, it } from "vitest";
|
|
2
|
+
import { __testing as bulkRetrieveTesting } from "../bulk-retrieve.js";
|
|
3
|
+
/**
|
|
4
|
+
* Phase 1.5 W1.5-01 + W1.5-02 unit tests.
|
|
5
|
+
*
|
|
6
|
+
* Contract under test:
|
|
7
|
+
* - The watchdog clock for a `failSoft`-wrapped source starts at SLOT
|
|
8
|
+
* ACQUISITION (body-entry of the wrapped async generator), not at
|
|
9
|
+
* source REGISTRATION. A wedged source must not consume the clock
|
|
10
|
+
* budget of queued-but-not-yet-executing neighbors.
|
|
11
|
+
* - When the watchdog fires for a wedged source, the merger's slot is
|
|
12
|
+
* released within ~one tick (the failSoft generator returns,
|
|
13
|
+
* mergeAsyncIterablesParallel sees `done:true`, advance(nextIdx) runs).
|
|
14
|
+
* - A namespaced warning is emitted into the shared `warnings` array.
|
|
15
|
+
* - The wedged iterator is parked into a background-wedge set; if it
|
|
16
|
+
* eventually yields, the record reaches the output stream with
|
|
17
|
+
* `attributes.lateYield === true`.
|
|
18
|
+
* - The background-wedge set is bounded by SFGRAPH_MAX_BACKGROUND_WEDGES.
|
|
19
|
+
* When the cap is exceeded, the oldest entry's stopWaiting flag is
|
|
20
|
+
* set and a `wedge:cap:backgroundWedgeAborted:...` warning fires.
|
|
21
|
+
*
|
|
22
|
+
* Tests run against `bulkRetrieve` end-to-end with a mock connection that
|
|
23
|
+
* causes specific sources to wedge. Real timers are used (not fake) so
|
|
24
|
+
* the watchdog's setTimeout interacts with real iterator promises; budgets
|
|
25
|
+
* are squeezed via env vars to keep the suite fast.
|
|
26
|
+
*
|
|
27
|
+
* Note on backgrounds-wedge cap test (test 3): the production cap is 4 by
|
|
28
|
+
* default; we set it to 2 via env var at module import time. We do NOT
|
|
29
|
+
* use vi.useFakeTimers() because the watchdog is implemented with real
|
|
30
|
+
* setTimeout — fake timers would prevent the iterator-vs-timeout race
|
|
31
|
+
* from advancing and the test would hang.
|
|
32
|
+
*/
|
|
33
|
+
/** Async generator: never yields, never resolves. Used as a wedge factory. */
|
|
34
|
+
async function* hangForever() {
|
|
35
|
+
await new Promise(() => {
|
|
36
|
+
/* hang */
|
|
37
|
+
});
|
|
38
|
+
yield {
|
|
39
|
+
ref: {
|
|
40
|
+
category: "ApexClass",
|
|
41
|
+
memberType: "ApexClass",
|
|
42
|
+
memberName: "Never",
|
|
43
|
+
lastModifiedAt: null,
|
|
44
|
+
sourceUri: "",
|
|
45
|
+
},
|
|
46
|
+
content: "",
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
/** Async generator: hangs for `wedgeMs` ms then yields one record, then
|
|
50
|
+
* ends. Used to simulate a source that eventually resolves AFTER the
|
|
51
|
+
* watchdog has already released its slot — the late-yield case. */
|
|
52
|
+
function delayedYield(wedgeMs, memberName) {
|
|
53
|
+
return async function* () {
|
|
54
|
+
await new Promise((r) => setTimeout(r, wedgeMs));
|
|
55
|
+
yield {
|
|
56
|
+
ref: {
|
|
57
|
+
category: "ApexClass",
|
|
58
|
+
memberType: "ApexClass",
|
|
59
|
+
memberName,
|
|
60
|
+
lastModifiedAt: null,
|
|
61
|
+
sourceUri: "",
|
|
62
|
+
},
|
|
63
|
+
content: "",
|
|
64
|
+
};
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
/** Async generator: yields N synthetic ApexClass records back-to-back, then
|
|
68
|
+
* ends. Used as a healthy neighbor source. */
|
|
69
|
+
function healthySource(count, prefix) {
|
|
70
|
+
return async function* () {
|
|
71
|
+
for (let i = 0; i < count; i += 1) {
|
|
72
|
+
yield {
|
|
73
|
+
ref: {
|
|
74
|
+
category: "ApexClass",
|
|
75
|
+
memberType: "ApexClass",
|
|
76
|
+
memberName: `${prefix}-${i}`,
|
|
77
|
+
lastModifiedAt: null,
|
|
78
|
+
sourceUri: "",
|
|
79
|
+
},
|
|
80
|
+
content: "",
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
describe("W1.5-01: watchdog clock starts at slot acquisition", () => {
|
|
86
|
+
it("does not advance clock for queued sources while a wedged source holds its slot", async () => {
|
|
87
|
+
// Setup: 2 sources, concurrency=1 (set via env), source A wedges,
|
|
88
|
+
// source B is queued. Watchdog firstYield budget is 200ms (test-only
|
|
89
|
+
// tight budget). Expectation: source B starts EXECUTING only after
|
|
90
|
+
// A's slot is released by the watchdog. B's own clock starts at that
|
|
91
|
+
// moment, NOT 200ms earlier when the merger was registered.
|
|
92
|
+
const warnings = [];
|
|
93
|
+
const sourceALabel = "test:A";
|
|
94
|
+
const sourceBLabel = "test:B";
|
|
95
|
+
const aStartRef = { value: 0 };
|
|
96
|
+
const bStartRef = { value: 0 };
|
|
97
|
+
// failSoft is not exported; use the testing entry point.
|
|
98
|
+
const merger = bulkRetrieveTesting.mergeAsyncIterablesParallel;
|
|
99
|
+
const failSoft = bulkRetrieveTesting.failSoft;
|
|
100
|
+
const makeCoordinator = bulkRetrieveTesting.createWedgeCoordinator;
|
|
101
|
+
const coordinator = makeCoordinator(warnings);
|
|
102
|
+
// Tight test budget: 150ms first-yield.
|
|
103
|
+
bulkRetrieveTesting.setTestWatchdogBudgets({ firstYieldMs: 150, inactivityMs: 60_000 });
|
|
104
|
+
try {
|
|
105
|
+
// Concurrency=1 so B has to wait for A's slot.
|
|
106
|
+
const sources = [
|
|
107
|
+
failSoft(sourceALabel, hangForever, undefined, coordinator, aStartRef),
|
|
108
|
+
failSoft(sourceBLabel, healthySource(2, "B"), undefined, coordinator, bStartRef),
|
|
109
|
+
];
|
|
110
|
+
const start = Date.now();
|
|
111
|
+
const out = [];
|
|
112
|
+
// Override concurrency via the merger's optional second arg (test
|
|
113
|
+
// hook): pass concurrency=1.
|
|
114
|
+
for await (const m of merger(sources, { concurrency: 1 })) {
|
|
115
|
+
out.push(m);
|
|
116
|
+
}
|
|
117
|
+
const elapsed = Date.now() - start;
|
|
118
|
+
// B's records should all be present (source A wedged, source B drained).
|
|
119
|
+
const bNames = out.map((m) => m.ref.memberName).filter((n) => n.startsWith("B-"));
|
|
120
|
+
expect(bNames.sort()).toEqual(["B-0", "B-1"]);
|
|
121
|
+
// Source A's clock should have started near `start` (it acquired the
|
|
122
|
+
// slot first).
|
|
123
|
+
expect(aStartRef.value).toBeGreaterThanOrEqual(start);
|
|
124
|
+
expect(aStartRef.value).toBeLessThan(start + 80);
|
|
125
|
+
// Source B's clock should have started AFTER A's watchdog fired —
|
|
126
|
+
// i.e., at least 150ms (firstYield budget) after A's start. NOT at
|
|
127
|
+
// registration time (which is `start`).
|
|
128
|
+
expect(bStartRef.value).toBeGreaterThanOrEqual(aStartRef.value + 140);
|
|
129
|
+
// Total elapsed: A wedges (~150ms) + B drains (~0ms) = ~150-300ms.
|
|
130
|
+
expect(elapsed).toBeLessThan(800);
|
|
131
|
+
// Exactly one wedge:test:A:firstYield:... warning fired.
|
|
132
|
+
const aWedges = warnings.filter((w) => w.startsWith(`wedge:${sourceALabel}:firstYield:`));
|
|
133
|
+
expect(aWedges.length).toBe(1);
|
|
134
|
+
}
|
|
135
|
+
finally {
|
|
136
|
+
bulkRetrieveTesting.resetTestWatchdogBudgets();
|
|
137
|
+
}
|
|
138
|
+
});
|
|
139
|
+
});
|
|
140
|
+
describe("W1.5-02: soft-isolate on wedge", () => {
|
|
141
|
+
it("releases slot, emits namespaced warning, runs queued neighbor", async () => {
|
|
142
|
+
const warnings = [];
|
|
143
|
+
const coordinator = bulkRetrieveTesting.createWedgeCoordinator(warnings);
|
|
144
|
+
bulkRetrieveTesting.setTestWatchdogBudgets({ firstYieldMs: 100, inactivityMs: 60_000 });
|
|
145
|
+
try {
|
|
146
|
+
const sources = [
|
|
147
|
+
bulkRetrieveTesting.failSoft("wedger", hangForever, undefined, coordinator),
|
|
148
|
+
bulkRetrieveTesting.failSoft("healthy", healthySource(3, "H"), undefined, coordinator),
|
|
149
|
+
];
|
|
150
|
+
const out = [];
|
|
151
|
+
for await (const m of bulkRetrieveTesting.mergeAsyncIterablesParallel(sources, {
|
|
152
|
+
concurrency: 1,
|
|
153
|
+
})) {
|
|
154
|
+
out.push(m);
|
|
155
|
+
}
|
|
156
|
+
// Healthy source drained completely.
|
|
157
|
+
const hCount = out.filter((m) => m.ref.memberName.startsWith("H-")).length;
|
|
158
|
+
expect(hCount).toBe(3);
|
|
159
|
+
// Exactly one wedge warning emitted with the right shape.
|
|
160
|
+
const w = warnings.find((s) => s.startsWith("wedge:wedger:firstYield:"));
|
|
161
|
+
expect(w).toBeTruthy();
|
|
162
|
+
// <stageDetail> is `${firstYieldMs/1000}s`; with a 100ms test budget
|
|
163
|
+
// that's `0.1s` — the regex must accept decimals.
|
|
164
|
+
expect(w).toMatch(/^wedge:wedger:firstYield:[\d.]+s:lastYielded=<none>:wedgedForMs=\d+$/);
|
|
165
|
+
// No error propagated up — for-await completed normally.
|
|
166
|
+
}
|
|
167
|
+
finally {
|
|
168
|
+
bulkRetrieveTesting.resetTestWatchdogBudgets();
|
|
169
|
+
}
|
|
170
|
+
});
|
|
171
|
+
});
|
|
172
|
+
describe("W1.5-02: background-wedge cap", () => {
|
|
173
|
+
it("sets stopWaiting=true on oldest entries when SFGRAPH_MAX_BACKGROUND_WEDGES is exceeded", async () => {
|
|
174
|
+
const warnings = [];
|
|
175
|
+
// Override the cap to 2 via the test-only setter.
|
|
176
|
+
const coordinator = bulkRetrieveTesting.createWedgeCoordinator(warnings, {
|
|
177
|
+
maxBackgroundWedges: 2,
|
|
178
|
+
});
|
|
179
|
+
bulkRetrieveTesting.setTestWatchdogBudgets({ firstYieldMs: 80, inactivityMs: 60_000 });
|
|
180
|
+
try {
|
|
181
|
+
// 4 wedging sources. With concurrency=1 they wedge sequentially:
|
|
182
|
+
// - W1 wedges at ~80ms (slot released, registered as bg-wedge #1)
|
|
183
|
+
// - W2 acquires slot, wedges at ~160ms (bg-wedge #2; cap not exceeded)
|
|
184
|
+
// - W3 acquires slot, wedges at ~240ms (would be bg-wedge #3, cap=2,
|
|
185
|
+
// so W1 evicted with stopWaiting=true)
|
|
186
|
+
// - W4 acquires slot, wedges at ~320ms (would be bg-wedge #4, cap=2,
|
|
187
|
+
// so W2 evicted with stopWaiting=true)
|
|
188
|
+
const sources = [
|
|
189
|
+
bulkRetrieveTesting.failSoft("w1", hangForever, undefined, coordinator),
|
|
190
|
+
bulkRetrieveTesting.failSoft("w2", hangForever, undefined, coordinator),
|
|
191
|
+
bulkRetrieveTesting.failSoft("w3", hangForever, undefined, coordinator),
|
|
192
|
+
bulkRetrieveTesting.failSoft("w4", hangForever, undefined, coordinator),
|
|
193
|
+
];
|
|
194
|
+
const out = [];
|
|
195
|
+
// Set the late-drain budget to 0 — we don't want to wait at the end.
|
|
196
|
+
coordinator.lateDrainBudgetMs = 0;
|
|
197
|
+
for await (const m of bulkRetrieveTesting.mergeAsyncIterablesParallel(sources, {
|
|
198
|
+
concurrency: 1,
|
|
199
|
+
})) {
|
|
200
|
+
out.push(m);
|
|
201
|
+
}
|
|
202
|
+
expect(out.length).toBe(0);
|
|
203
|
+
expect(coordinator.backgroundWedges.length).toBe(4);
|
|
204
|
+
// Find each entry by label.
|
|
205
|
+
const byLabel = new Map(coordinator.backgroundWedges.map((e) => [e.label, e]));
|
|
206
|
+
// Cap=2, with 4 wedges admitted in sequence: at the moment W3 is
|
|
207
|
+
// admitted the active count would become 3; oldest (W1) is evicted.
|
|
208
|
+
// When W4 is admitted the active count would become 3 again
|
|
209
|
+
// (W2 + W3 + W4); oldest active (W2) is evicted.
|
|
210
|
+
expect(byLabel.get("w1")?.stopWaitingRef.value).toBe(true);
|
|
211
|
+
expect(byLabel.get("w2")?.stopWaitingRef.value).toBe(true);
|
|
212
|
+
expect(byLabel.get("w3")?.stopWaitingRef.value).toBe(false);
|
|
213
|
+
expect(byLabel.get("w4")?.stopWaitingRef.value).toBe(false);
|
|
214
|
+
// Exactly 2 backgroundWedgeAborted warnings — one per eviction.
|
|
215
|
+
const aborts = warnings.filter((w) => w.startsWith("wedge:cap:backgroundWedgeAborted:"));
|
|
216
|
+
expect(aborts.length).toBe(2);
|
|
217
|
+
// First eviction names w1; second names w2.
|
|
218
|
+
expect(aborts[0]).toContain("source=w1");
|
|
219
|
+
expect(aborts[1]).toContain("source=w2");
|
|
220
|
+
}
|
|
221
|
+
finally {
|
|
222
|
+
bulkRetrieveTesting.resetTestWatchdogBudgets();
|
|
223
|
+
}
|
|
224
|
+
});
|
|
225
|
+
});
|
|
226
|
+
describe("W1.5-02: late-yield drain", () => {
|
|
227
|
+
it("drains late records with attributes.lateYield=true and emits resolvedLate warning", async () => {
|
|
228
|
+
const warnings = [];
|
|
229
|
+
const coordinator = bulkRetrieveTesting.createWedgeCoordinator(warnings);
|
|
230
|
+
// Late-drain budget: 600ms — generous enough to catch the delayed yield.
|
|
231
|
+
coordinator.lateDrainBudgetMs = 600;
|
|
232
|
+
bulkRetrieveTesting.setTestWatchdogBudgets({ firstYieldMs: 100, inactivityMs: 60_000 });
|
|
233
|
+
try {
|
|
234
|
+
// Wedger first-yields after 250ms — well past the 100ms watchdog —
|
|
235
|
+
// so it gets parked into background. The late-drain should observe
|
|
236
|
+
// its first record arriving and tag it.
|
|
237
|
+
const sources = [
|
|
238
|
+
bulkRetrieveTesting.failSoft("tardy", delayedYield(250, "T-late"), undefined, coordinator),
|
|
239
|
+
bulkRetrieveTesting.failSoft("fast", healthySource(1, "F"), undefined, coordinator),
|
|
240
|
+
];
|
|
241
|
+
const out = [];
|
|
242
|
+
for await (const m of bulkRetrieveTesting.mergeAsyncIterablesParallel(sources, {
|
|
243
|
+
concurrency: 1,
|
|
244
|
+
})) {
|
|
245
|
+
out.push(m);
|
|
246
|
+
}
|
|
247
|
+
// The drainBackgroundWedge pass happens inside bulkRetrieve, not the
|
|
248
|
+
// merger. The merger test here doesn't run that pass — invoke it
|
|
249
|
+
// explicitly via the testing handle.
|
|
250
|
+
for (const entry of coordinator.backgroundWedges) {
|
|
251
|
+
for await (const m of bulkRetrieveTesting.drainBackgroundWedge(entry, coordinator.lateDrainBudgetMs, coordinator.warnings)) {
|
|
252
|
+
out.push(m);
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
// Fast source's record is present, plus the late record with the tag.
|
|
256
|
+
const lateRecord = out.find((m) => m.ref.memberName === "T-late");
|
|
257
|
+
expect(lateRecord).toBeTruthy();
|
|
258
|
+
expect(lateRecord?.attributes?.lateYield).toBe(true);
|
|
259
|
+
// resolvedLate warning emitted.
|
|
260
|
+
const resolved = warnings.find((w) => w.startsWith("wedge:tardy:resolvedLate:"));
|
|
261
|
+
expect(resolved).toBeTruthy();
|
|
262
|
+
expect(resolved).toContain("records=1");
|
|
263
|
+
// No error propagated.
|
|
264
|
+
}
|
|
265
|
+
finally {
|
|
266
|
+
bulkRetrieveTesting.resetTestWatchdogBudgets();
|
|
267
|
+
}
|
|
268
|
+
});
|
|
269
|
+
});
|
|
270
|
+
//# sourceMappingURL=wedge-isolation.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"wedge-isolation.test.js","sourceRoot":"","sources":["../../../../src/extractors/live-org/__tests__/wedge-isolation.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAE9C,OAAO,EAAE,SAAS,IAAI,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAEvE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAEH,8EAA8E;AAC9E,KAAK,SAAS,CAAC,CAAC,WAAW;IACzB,MAAM,IAAI,OAAO,CAAO,GAAG,EAAE;QAC3B,UAAU;IACZ,CAAC,CAAC,CAAC;IACH,MAAM;QACJ,GAAG,EAAE;YACH,QAAQ,EAAE,WAAoB;YAC9B,UAAU,EAAE,WAAW;YACvB,UAAU,EAAE,OAAO;YACnB,cAAc,EAAE,IAAI;YACpB,SAAS,EAAE,EAAE;SACd;QACD,OAAO,EAAE,EAAE;KACZ,CAAC;AACJ,CAAC;AAED;;oEAEoE;AACpE,SAAS,YAAY,CAAC,OAAe,EAAE,UAAkB;IACvD,OAAO,KAAK,SAAS,CAAC;QACpB,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;QACjD,MAAM;YACJ,GAAG,EAAE;gBACH,QAAQ,EAAE,WAAoB;gBAC9B,UAAU,EAAE,WAAW;gBACvB,UAAU;gBACV,cAAc,EAAE,IAAI;gBACpB,SAAS,EAAE,EAAE;aACd;YACD,OAAO,EAAE,EAAE;SACZ,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC;AAED;+CAC+C;AAC/C,SAAS,aAAa,CAAC,KAAa,EAAE,MAAc;IAClD,OAAO,KAAK,SAAS,CAAC;QACpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;YAClC,MAAM;gBACJ,GAAG,EAAE;oBACH,QAAQ,EAAE,WAAoB;oBAC9B,UAAU,EAAE,WAAW;oBACvB,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,EAAE;oBAC5B,cAAc,EAAE,IAAI;oBACpB,SAAS,EAAE,EAAE;iBACd;gBACD,OAAO,EAAE,EAAE;aACZ,CAAC;QACJ,CAAC;IACH,CAAC,CAAC;AACJ,CAAC;AAED,QAAQ,CAAC,oDAAoD,EAAE,GAAG,EAAE;IAClE,EAAE,CAAC,gFAAgF,EAAE,KAAK,IAAI,EAAE;QAC9F,kEAAkE;QAClE,qEAAqE;QACrE,mEAAmE;QACnE,qEAAqE;QACrE,4DAA4D;QAC5D,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,MAAM,YAAY,GAAG,QAAQ,CAAC;QAC9B,MAAM,YAAY,GAAG,QAAQ,CAAC;QAE9B,MAAM,SAAS,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QAC/B,MAAM,SAAS,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QAE/B,yDAAyD;QACzD,MAAM,MAAM,GAAG,mBAAmB,CAAC,2BAA2B,CAAC;QAC/D,MAAM,QAAQ,GAAG,mBAAmB,CAAC,QAAQ,CAAC;QAC9C,MAAM,eAAe,GAAG,mBAAmB,CAAC,sBAAsB,CAAC;QAEnE,MAAM,WAAW,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAC;QAC9C,wCAAwC;QACxC,mBAAmB,CAAC,sBAAsB,CAAC,EAAE,YAAY,EAAE,GAAG,EAAE,YAAY,EAAE,MAAM,EAAE,CAAC,CAAC;QAExF,IAAI,CAAC;YACH,+CAA+C;YAC/C,MAAM,OAAO,GAAG;gBACd,QAAQ,CAAC,YAAY,EAAE,WAAW,EAAE,SAAS,EAAE,WAAW,EAAE,SAAS,CAAC;gBACtE,QAAQ,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,SAAS,EAAE,WAAW,EAAE,SAAS,CAAC;aACjF,CAAC;YACF,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACzB,MAAM,GAAG,GAAgB,EAAE,CAAC;YAC5B,kEAAkE;YAClE,6BAA6B;YAC7B,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC;gBAC1D,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACd,CAAC;YACD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;YAEnC,yEAAyE;YACzE,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC;YAClF,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;YAE9C,qEAAqE;YACrE,eAAe;YACf,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,sBAAsB,CAAC,KAAK,CAAC,CAAC;YACtD,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;YAEjD,kEAAkE;YAClE,mEAAmE;YACnE,wCAAwC;YACxC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,sBAAsB,CAAC,SAAS,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC;YAEtE,mEAAmE;YACnE,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;YAElC,yDAAyD;YACzD,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,SAAS,YAAY,cAAc,CAAC,CAAC,CAAC;YAC1F,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACjC,CAAC;gBAAS,CAAC;YACT,mBAAmB,CAAC,wBAAwB,EAAE,CAAC;QACjD,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,gCAAgC,EAAE,GAAG,EAAE;IAC9C,EAAE,CAAC,+DAA+D,EAAE,KAAK,IAAI,EAAE;QAC7E,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,MAAM,WAAW,GAAG,mBAAmB,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC;QACzE,mBAAmB,CAAC,sBAAsB,CAAC,EAAE,YAAY,EAAE,GAAG,EAAE,YAAY,EAAE,MAAM,EAAE,CAAC,CAAC;QAExF,IAAI,CAAC;YACH,MAAM,OAAO,GAAG;gBACd,mBAAmB,CAAC,QAAQ,CAAC,QAAQ,EAAE,WAAW,EAAE,SAAS,EAAE,WAAW,CAAC;gBAC3E,mBAAmB,CAAC,QAAQ,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,SAAS,EAAE,WAAW,CAAC;aACvF,CAAC;YACF,MAAM,GAAG,GAAgB,EAAE,CAAC;YAC5B,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,mBAAmB,CAAC,2BAA2B,CAAC,OAAO,EAAE;gBAC7E,WAAW,EAAE,CAAC;aACf,CAAC,EAAE,CAAC;gBACH,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACd,CAAC;YAED,qCAAqC;YACrC,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;YAC3E,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAEvB,0DAA0D;YAC1D,MAAM,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,0BAA0B,CAAC,CAAC,CAAC;YACzE,MAAM,CAAC,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC;YACvB,qEAAqE;YACrE,kDAAkD;YAClD,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,sEAAsE,CAAC,CAAC;YAE1F,yDAAyD;QAC3D,CAAC;gBAAS,CAAC;YACT,mBAAmB,CAAC,wBAAwB,EAAE,CAAC;QACjD,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,+BAA+B,EAAE,GAAG,EAAE;IAC7C,EAAE,CAAC,wFAAwF,EAAE,KAAK,IAAI,EAAE;QACtG,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,kDAAkD;QAClD,MAAM,WAAW,GAAG,mBAAmB,CAAC,sBAAsB,CAAC,QAAQ,EAAE;YACvE,mBAAmB,EAAE,CAAC;SACvB,CAAC,CAAC;QACH,mBAAmB,CAAC,sBAAsB,CAAC,EAAE,YAAY,EAAE,EAAE,EAAE,YAAY,EAAE,MAAM,EAAE,CAAC,CAAC;QAEvF,IAAI,CAAC;YACH,iEAAiE;YACjE,kEAAkE;YAClE,uEAAuE;YACvE,qEAAqE;YACrE,yCAAyC;YACzC,qEAAqE;YACrE,yCAAyC;YACzC,MAAM,OAAO,GAAG;gBACd,mBAAmB,CAAC,QAAQ,CAAC,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,WAAW,CAAC;gBACvE,mBAAmB,CAAC,QAAQ,CAAC,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,WAAW,CAAC;gBACvE,mBAAmB,CAAC,QAAQ,CAAC,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,WAAW,CAAC;gBACvE,mBAAmB,CAAC,QAAQ,CAAC,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,WAAW,CAAC;aACxE,CAAC;YACF,MAAM,GAAG,GAAgB,EAAE,CAAC;YAC5B,qEAAqE;YACrE,WAAW,CAAC,iBAAiB,GAAG,CAAC,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,mBAAmB,CAAC,2BAA2B,CAAC,OAAO,EAAE;gBAC7E,WAAW,EAAE,CAAC;aACf,CAAC,EAAE,CAAC;gBACH,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACd,CAAC;YACD,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC3B,MAAM,CAAC,WAAW,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAEpD,4BAA4B;YAC5B,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/E,iEAAiE;YACjE,oEAAoE;YACpE,4DAA4D;YAC5D,iDAAiD;YACjD,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,cAAc,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC3D,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,cAAc,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC3D,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,cAAc,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC5D,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,cAAc,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAE5D,gEAAgE;YAChE,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,mCAAmC,CAAC,CAAC,CAAC;YACzF,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC9B,4CAA4C;YAC5C,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;YACzC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;QAC3C,CAAC;gBAAS,CAAC;YACT,mBAAmB,CAAC,wBAAwB,EAAE,CAAC;QACjD,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,2BAA2B,EAAE,GAAG,EAAE;IACzC,EAAE,CAAC,mFAAmF,EAAE,KAAK,IAAI,EAAE;QACjG,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,MAAM,WAAW,GAAG,mBAAmB,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC;QACzE,yEAAyE;QACzE,WAAW,CAAC,iBAAiB,GAAG,GAAG,CAAC;QACpC,mBAAmB,CAAC,sBAAsB,CAAC,EAAE,YAAY,EAAE,GAAG,EAAE,YAAY,EAAE,MAAM,EAAE,CAAC,CAAC;QAExF,IAAI,CAAC;YACH,mEAAmE;YACnE,mEAAmE;YACnE,wCAAwC;YACxC,MAAM,OAAO,GAAG;gBACd,mBAAmB,CAAC,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAC,GAAG,EAAE,QAAQ,CAAC,EAAE,SAAS,EAAE,WAAW,CAAC;gBAC1F,mBAAmB,CAAC,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,SAAS,EAAE,WAAW,CAAC;aACpF,CAAC;YACF,MAAM,GAAG,GAAgB,EAAE,CAAC;YAC5B,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,mBAAmB,CAAC,2BAA2B,CAAC,OAAO,EAAE;gBAC7E,WAAW,EAAE,CAAC;aACf,CAAC,EAAE,CAAC;gBACH,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACd,CAAC;YAED,qEAAqE;YACrE,iEAAiE;YACjE,qCAAqC;YACrC,KAAK,MAAM,KAAK,IAAI,WAAW,CAAC,gBAAgB,EAAE,CAAC;gBACjD,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,mBAAmB,CAAC,oBAAoB,CAC5D,KAAK,EACL,WAAW,CAAC,iBAAiB,EAC7B,WAAW,CAAC,QAAQ,CACrB,EAAE,CAAC;oBACF,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACd,CAAC;YACH,CAAC;YAED,sEAAsE;YACtE,MAAM,UAAU,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC;YAClE,MAAM,CAAC,UAAU,CAAC,CAAC,UAAU,EAAE,CAAC;YAChC,MAAM,CAAC,UAAU,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAErD,gCAAgC;YAChC,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,2BAA2B,CAAC,CAAC,CAAC;YACjF,MAAM,CAAC,QAAQ,CAAC,CAAC,UAAU,EAAE,CAAC;YAC9B,MAAM,CAAC,QAAQ,CAAC,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;YAExC,uBAAuB;QACzB,CAAC;gBAAS,CAAC;YACT,mBAAmB,CAAC,wBAAwB,EAAE,CAAC;QACjD,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -12,6 +12,70 @@ export interface IngestSkipReport {
|
|
|
12
12
|
category: SkipCategory;
|
|
13
13
|
}>;
|
|
14
14
|
}
|
|
15
|
+
/**
|
|
16
|
+
* Phase 1.5 (W1.5-02): soft-isolate wedge coordination state.
|
|
17
|
+
*
|
|
18
|
+
* Threaded through the bulkRetrieve fan-out so `failSoft` wrappers can:
|
|
19
|
+
* (a) push namespaced-string warnings into a shared sink (the warnings
|
|
20
|
+
* eventually surface on LiveIngestResult.warnings),
|
|
21
|
+
* (b) park the still-pending iterator from a wedged source into a bounded
|
|
22
|
+
* background-wedge set (the merger's slot is released immediately,
|
|
23
|
+
* so queued neighbors can proceed),
|
|
24
|
+
* (c) yield late-arriving records from those background iterators with
|
|
25
|
+
* `attributes.lateYield = true` so they still reach the output stream.
|
|
26
|
+
*
|
|
27
|
+
* One coordinator instance per bulkRetrieve invocation. Default sourced
|
|
28
|
+
* from `SFGRAPH_MAX_BACKGROUND_WEDGES` (default 4).
|
|
29
|
+
*/
|
|
30
|
+
export interface BackgroundWedgeEntry {
|
|
31
|
+
label: string;
|
|
32
|
+
/** Iterator that was wedged. Used by the late-drain loop to keep pumping
|
|
33
|
+
* it after slot release. */
|
|
34
|
+
iterator: AsyncIterator<RawMember>;
|
|
35
|
+
/**
|
|
36
|
+
* The in-flight `it.next()` promise that was racing the watchdog at the
|
|
37
|
+
* moment the watchdog won. Async-generator semantics queue concurrent
|
|
38
|
+
* `.next()` calls — so if the drain loop calls `iterator.next()` while
|
|
39
|
+
* this promise is still pending, the drain will wait behind it AND
|
|
40
|
+
* "lose" the value it eventually resolves with (the first `.next()` gets
|
|
41
|
+
* it, then the drain's `.next()` advances to the NEXT yield). We must
|
|
42
|
+
* therefore consume this promise BEFORE calling `iterator.next()` again
|
|
43
|
+
* in the drain loop. After the drain consumes the first pending result,
|
|
44
|
+
* this field is set to null and subsequent calls go through the normal
|
|
45
|
+
* `iterator.next()` path.
|
|
46
|
+
*/
|
|
47
|
+
pendingFirstNext: Promise<IteratorResult<RawMember>> | null;
|
|
48
|
+
/** Wall-clock epoch when this entry was registered (post slot-release). */
|
|
49
|
+
registeredAt: number;
|
|
50
|
+
/** Set true when the cap-eviction policy decides this wedge has outlived
|
|
51
|
+
* its budget. The drain loop checks this each iteration and bails. The
|
|
52
|
+
* pending HTTP request (if any) is NOT actively cancelled — that's the
|
|
53
|
+
* job of W1.5-03. */
|
|
54
|
+
stopWaitingRef: {
|
|
55
|
+
value: boolean;
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
export interface WedgeCoordinator {
|
|
59
|
+
/** Shared namespaced-warning sink. Format documented in PLAN.md:
|
|
60
|
+
* `wedge:<source>:<stage>:<detail>`. The warning strings reach the
|
|
61
|
+
* surface via `bulkRetrieve` opts → `LiveIngestResult.warnings`. */
|
|
62
|
+
warnings: string[];
|
|
63
|
+
/** Active background-wedge set. Bounded by `maxBackgroundWedges`. When
|
|
64
|
+
* a new wedge would exceed the cap, the OLDEST entry gets its
|
|
65
|
+
* `stopWaitingRef.value = true` flag set + a `backgroundWedgeAborted`
|
|
66
|
+
* warning is emitted. The old iterator is NOT removed from the set —
|
|
67
|
+
* the late-drain loop sees the flag and bails. */
|
|
68
|
+
backgroundWedges: BackgroundWedgeEntry[];
|
|
69
|
+
/** Maximum simultaneous active background wedges. From
|
|
70
|
+
* `SFGRAPH_MAX_BACKGROUND_WEDGES`, default 4. */
|
|
71
|
+
maxBackgroundWedges: number;
|
|
72
|
+
/** Bounded wall-clock budget for the post-merger late-drain pass. The
|
|
73
|
+
* drain races each pending `it.next()` against this budget; whatever
|
|
74
|
+
* arrives gets `attributes.lateYield = true`. Default 60 000ms; set to
|
|
75
|
+
* 0 to disable late-drain entirely (used by some tests). */
|
|
76
|
+
lateDrainBudgetMs: number;
|
|
77
|
+
}
|
|
78
|
+
declare function createWedgeCoordinator(warnings?: string[], overrides?: Partial<Pick<WedgeCoordinator, "maxBackgroundWedges" | "lateDrainBudgetMs">>): WedgeCoordinator;
|
|
15
79
|
export type SkipCategory = "insufficient_access" | "not_found" | "rate_limit" | "network" | "unknown";
|
|
16
80
|
/** Naive sequential merge — predictable order, simpler back-pressure semantics.
|
|
17
81
|
* Kept for back-compat and tests; production ingest uses
|
|
@@ -38,7 +102,71 @@ export declare function mergeAsyncIterables<T>(...iters: Array<AsyncIterable<T>>
|
|
|
38
102
|
* Output ordering is non-deterministic. `live-ingest`'s processOne is
|
|
39
103
|
* order-independent (idempotent per-record upserts), so this is safe.
|
|
40
104
|
*/
|
|
105
|
+
export interface MergeParallelOptions {
|
|
106
|
+
/** Override the sliding-window size for this call. When omitted, the
|
|
107
|
+
* env-var / default path applies as before. Used by unit tests that
|
|
108
|
+
* need deterministic queue-vs-active ordering. */
|
|
109
|
+
concurrency?: number;
|
|
110
|
+
}
|
|
41
111
|
export declare function mergeAsyncIterablesParallel<T>(...iters: Array<AsyncIterable<T>>): AsyncIterable<T>;
|
|
112
|
+
export declare function mergeAsyncIterablesParallel<T>(iters: Array<AsyncIterable<T>>, opts: MergeParallelOptions): AsyncIterable<T>;
|
|
113
|
+
/** Read watchdog budgets per-call so unit tests can override via env vars
|
|
114
|
+
* without restarting the process. Production env should leave these unset:
|
|
115
|
+
* ordinary sources get the documented 90s first-yield / 5min inactivity;
|
|
116
|
+
* a source in SOURCE_WATCHDOG_OVERRIDES gets its scoped budget instead. */
|
|
117
|
+
declare function readWatchdogBudgets(label?: string): {
|
|
118
|
+
firstYieldMs: number;
|
|
119
|
+
inactivityMs: number;
|
|
120
|
+
};
|
|
121
|
+
/**
|
|
122
|
+
* Drain a single background-wedge iterator. Called once per wedged source,
|
|
123
|
+
* post-merger, with a wall-clock budget. Each iteration races the iterator's
|
|
124
|
+
* `it.next()` against the remaining budget. Records arriving inside the
|
|
125
|
+
* budget are yielded with `attributes.lateYield = true`. The pending HTTP
|
|
126
|
+
* request (if the wedge is on a network call) is NOT actively cancelled
|
|
127
|
+
* (W1.5-03's job); we just stop awaiting it.
|
|
128
|
+
*
|
|
129
|
+
* Bails immediately if `stopWaitingRef.value === true` (cap eviction).
|
|
130
|
+
*/
|
|
131
|
+
declare function drainBackgroundWedge(entry: BackgroundWedgeEntry, budgetMs: number, warnings: string[]): AsyncIterable<RawMember>;
|
|
132
|
+
/**
|
|
133
|
+
* Wrap an iterable so a thrown error is captured + the stream ends cleanly
|
|
134
|
+
* instead of aborting the whole ingest. The error is recorded into a
|
|
135
|
+
* shared skip report (consumed at end-of-run) and a compact ✗ line is
|
|
136
|
+
* printed so the user sees something happened without the full error
|
|
137
|
+
* message scrolling past during the run.
|
|
138
|
+
*
|
|
139
|
+
* Phase 1.5 W1.5-01 + W1.5-02 contract:
|
|
140
|
+
*
|
|
141
|
+
* - The watchdog clock starts when this generator's BODY first executes
|
|
142
|
+
* (i.e., when the merger calls `.next()` on us for the first time —
|
|
143
|
+
* that's slot-acquisition time). Queued sources still inside
|
|
144
|
+
* `mergeAsyncIterablesParallel`'s waiting list accumulate ZERO clock
|
|
145
|
+
* time. Implemented via an internal `startedAtRef` that's written once
|
|
146
|
+
* on body entry; if the caller passes a `startedAtRef` we update it in
|
|
147
|
+
* place so the merger can observe the slot-acquisition moment.
|
|
148
|
+
*
|
|
149
|
+
* - On watchdog fire, we DO NOT throw (which would abort the merger's
|
|
150
|
+
* Promise.race for our slot AND lose the still-pending iterator).
|
|
151
|
+
* Instead we:
|
|
152
|
+
* 1. Emit a `wedge:<label>:firstYield:90s:...` (or `:inactivity:300s:`)
|
|
153
|
+
* warning into coordinator.warnings.
|
|
154
|
+
* 2. Register the still-pending iterator into coordinator.background
|
|
155
|
+
* Wedges with a stopWaitingRef. Enforce the cap (evict oldest).
|
|
156
|
+
* 3. RETURN from the generator. This cleanly releases the merger's
|
|
157
|
+
* slot — `pending.delete(idx)` runs in mergeAsyncIterablesParallel
|
|
158
|
+
* and the next queued source acquires the slot.
|
|
159
|
+
* Post-merger, the bulkRetrieve top-level fan-out drains the background
|
|
160
|
+
* wedge set; records arriving inside the late-drain budget are yielded
|
|
161
|
+
* with `attributes.lateYield = true`.
|
|
162
|
+
*/
|
|
163
|
+
declare function failSoft<T extends RawMember>(label: string, factory: () => AsyncIterable<T>, onError?: (label: string, err: Error) => void, coordinator?: WedgeCoordinator,
|
|
164
|
+
/** Optional outer-visible clock cell. The merger does not currently read
|
|
165
|
+
* this, but the contract is "set by failSoft on body entry"; tests use
|
|
166
|
+
* it to assert slot-acquisition timing. */
|
|
167
|
+
startedAtRef?: {
|
|
168
|
+
value: number;
|
|
169
|
+
}): AsyncIterable<T>;
|
|
42
170
|
export interface BulkRetrieveOpts {
|
|
43
171
|
skipReport?: IngestSkipReport;
|
|
44
172
|
/** When set, only invoke source labels in this set. Labels are the same
|
|
@@ -46,6 +174,54 @@ export interface BulkRetrieveOpts {
|
|
|
46
174
|
* 'security', 'integration', 'vlocity', 'omnistudio', or
|
|
47
175
|
* 'generic:<MetadataType>' for the long tail. */
|
|
48
176
|
onlyLabels?: Set<string>;
|
|
177
|
+
/** Opt-in: invoke the OmniStudio-on-Core retrieve() extractor in addition
|
|
178
|
+
* to the existing SOQL path. Default off — retrieve() consumes Metadata
|
|
179
|
+
* API quota (10k/24h) and is slower; only enable when the higher-fidelity
|
|
180
|
+
* XML envelope is actually needed by a downstream parser. */
|
|
181
|
+
enableOmnistudioRetrieve?: boolean;
|
|
182
|
+
/** API version string for the retrieve() request envelope. */
|
|
183
|
+
apiVersion?: string;
|
|
184
|
+
/**
|
|
185
|
+
* Phase 1.5 W1.5-02 warning sink. When provided, the watchdog soft-isolate
|
|
186
|
+
* path pushes namespaced wedge strings (`wedge:<source>:<stage>:...`) into
|
|
187
|
+
* this array. The caller (live-ingest) concatenates them onto
|
|
188
|
+
* LiveIngestResult.warnings. When omitted, wedge events fall through to
|
|
189
|
+
* the legacy `onSkip` / skipReport path so existing tests still see a
|
|
190
|
+
* skip entry.
|
|
191
|
+
*/
|
|
192
|
+
warnings?: string[];
|
|
49
193
|
}
|
|
50
194
|
export declare function bulkRetrieve(conn: any, caps: OrgCapabilities, orgId: OrgId, opts?: BulkRetrieveOpts | IngestSkipReport): AsyncIterable<RawMember>;
|
|
195
|
+
/**
|
|
196
|
+
* Internal-surface exports for unit tests in `__tests__/`. Not part of the
|
|
197
|
+
* public package API — these are intentionally namespaced under `__testing`
|
|
198
|
+
* so they don't appear in the package's exports surface to downstream
|
|
199
|
+
* consumers. (They're still importable, but the underscore convention
|
|
200
|
+
* signals "test only".)
|
|
201
|
+
*
|
|
202
|
+
* Exposed:
|
|
203
|
+
* - `failSoft`: wrap an iterable with the W1.5-01/02 watchdog + soft-isolate.
|
|
204
|
+
* - `createWedgeCoordinator`: build a coordinator + warnings sink with
|
|
205
|
+
* optional cap/drain overrides for deterministic tests.
|
|
206
|
+
* - `drainBackgroundWedge`: run the post-merger late-drain pass against
|
|
207
|
+
* a single background-wedge entry.
|
|
208
|
+
* - `mergeAsyncIterablesParallel`: re-export for symmetry; the (array,
|
|
209
|
+
* opts) overload is what tests use to fix concurrency.
|
|
210
|
+
* - `setTestWatchdogBudgets` / `resetTestWatchdogBudgets`: env-var-backed
|
|
211
|
+
* knobs to squeeze the watchdog budget for fast tests. Always paired
|
|
212
|
+
* via try/finally in tests so the environment is left clean.
|
|
213
|
+
*/
|
|
214
|
+
export declare const __testing: {
|
|
215
|
+
failSoft: typeof failSoft;
|
|
216
|
+
createWedgeCoordinator: typeof createWedgeCoordinator;
|
|
217
|
+
drainBackgroundWedge: typeof drainBackgroundWedge;
|
|
218
|
+
mergeAsyncIterablesParallel: typeof mergeAsyncIterablesParallel;
|
|
219
|
+
readWatchdogBudgets: typeof readWatchdogBudgets;
|
|
220
|
+
setTestWatchdogBudgets(opts: {
|
|
221
|
+
firstYieldMs: number;
|
|
222
|
+
inactivityMs: number;
|
|
223
|
+
}): void;
|
|
224
|
+
resetTestWatchdogBudgets(): void;
|
|
225
|
+
};
|
|
226
|
+
export {};
|
|
51
227
|
//# sourceMappingURL=bulk-retrieve.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"bulk-retrieve.d.ts","sourceRoot":"","sources":["../../../src/extractors/live-org/bulk-retrieve.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,6BAA6B,CAAC;AACzD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kCAAkC,CAAC;AAClE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"bulk-retrieve.d.ts","sourceRoot":"","sources":["../../../src/extractors/live-org/bulk-retrieve.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,6BAA6B,CAAC;AACzD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kCAAkC,CAAC;AAClE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAezD;;;wCAGwC;AACxC,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,YAAY,CAAA;KAAE,CAAC,CAAC;CACzE;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd;iCAC6B;IAC7B,QAAQ,EAAE,aAAa,CAAC,SAAS,CAAC,CAAC;IACnC;;;;;;;;;;;OAWG;IACH,gBAAgB,EAAE,OAAO,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC,GAAG,IAAI,CAAC;IAC5D,2EAA2E;IAC3E,YAAY,EAAE,MAAM,CAAC;IACrB;;;0BAGsB;IACtB,cAAc,EAAE;QAAE,KAAK,EAAE,OAAO,CAAA;KAAE,CAAC;CACpC;AAED,MAAM,WAAW,gBAAgB;IAC/B;;yEAEqE;IACrE,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB;;;;uDAImD;IACnD,gBAAgB,EAAE,oBAAoB,EAAE,CAAC;IACzC;sDACkD;IAClD,mBAAmB,EAAE,MAAM,CAAC;IAC5B;;;iEAG6D;IAC7D,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED,iBAAS,sBAAsB,CAC7B,QAAQ,GAAE,MAAM,EAAO,EACvB,SAAS,CAAC,EAAE,OAAO,CAAC,IAAI,CAAC,gBAAgB,EAAE,qBAAqB,GAAG,mBAAmB,CAAC,CAAC,GACvF,gBAAgB,CAoBlB;AAED,MAAM,MAAM,YAAY,GACpB,qBAAqB,GACrB,WAAW,GACX,YAAY,GACZ,SAAS,GACT,SAAS,CAAC;AAgBd;;;4EAG4E;AAC5E,wBAAuB,mBAAmB,CAAC,CAAC,EAAE,GAAG,KAAK,EAAE,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,CAIjG;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,WAAW,oBAAoB;IACnC;;uDAEmD;IACnD,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,wBAAgB,2BAA2B,CAAC,CAAC,EAAE,GAAG,KAAK,EAAE,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;AACpG,wBAAgB,2BAA2B,CAAC,CAAC,EAC3C,KAAK,EAAE,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,EAC9B,IAAI,EAAE,oBAAoB,GACzB,aAAa,CAAC,CAAC,CAAC,CAAC;AA0HpB;;;4EAG4E;AAC5E,iBAAS,mBAAmB,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG;IAAE,YAAY,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,CAc3F;AAgDD;;;;;;;;;GASG;AACH,iBAAgB,oBAAoB,CAClC,KAAK,EAAE,oBAAoB,EAC3B,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAAE,GACjB,aAAa,CAAC,SAAS,CAAC,CAiE1B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,iBAAgB,QAAQ,CAAC,CAAC,SAAS,SAAS,EAC1C,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,aAAa,CAAC,CAAC,CAAC,EAC/B,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,KAAK,IAAI,EAC7C,WAAW,CAAC,EAAE,gBAAgB;AAC9B;;4CAE4C;AAC5C,YAAY,CAAC,EAAE;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,GAC/B,aAAa,CAAC,CAAC,CAAC,CA2JlB;AA+ID,MAAM,WAAW,gBAAgB;IAC/B,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B;;;sDAGkD;IAClD,UAAU,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACzB;;;kEAG8D;IAC9D,wBAAwB,CAAC,EAAE,OAAO,CAAC;IACnC,8DAA8D;IAC9D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;;;;;OAOG;IACH,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,wBAAuB,YAAY,CACjC,IAAI,EAAE,GAAG,EACT,IAAI,EAAE,eAAe,EACrB,KAAK,EAAE,KAAK,EACZ,IAAI,GAAE,gBAAgB,GAAG,gBAAqB,GAC7C,aAAa,CAAC,SAAS,CAAC,CAsM1B;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,eAAO,MAAM,SAAS;;;;;;iCAMS;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;gCAItD,IAAI;CASjC,CAAC"}
|