@oscharko-dev/keiko 0.3.14 → 0.3.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ui/csp-hashes.json +6 -6
- package/dist/ui/static/404.html +1 -1
- package/dist/ui/static/__next.__PAGE__.txt +1 -1
- package/dist/ui/static/__next._full.txt +1 -1
- package/dist/ui/static/__next._head.txt +1 -1
- package/dist/ui/static/__next._index.txt +1 -1
- package/dist/ui/static/__next._tree.txt +1 -1
- package/dist/ui/static/_not-found/__next._full.txt +1 -1
- package/dist/ui/static/_not-found/__next._head.txt +1 -1
- package/dist/ui/static/_not-found/__next._index.txt +1 -1
- package/dist/ui/static/_not-found/__next._not-found.__PAGE__.txt +1 -1
- package/dist/ui/static/_not-found/__next._not-found.txt +1 -1
- package/dist/ui/static/_not-found/__next._tree.txt +1 -1
- package/dist/ui/static/_not-found.html +1 -1
- package/dist/ui/static/_not-found.txt +1 -1
- package/dist/ui/static/atlassian-connectors/__next._full.txt +1 -1
- package/dist/ui/static/atlassian-connectors/__next._head.txt +1 -1
- package/dist/ui/static/atlassian-connectors/__next._index.txt +1 -1
- package/dist/ui/static/atlassian-connectors/__next._tree.txt +1 -1
- package/dist/ui/static/atlassian-connectors/__next.atlassian-connectors.__PAGE__.txt +1 -1
- package/dist/ui/static/atlassian-connectors/__next.atlassian-connectors.txt +1 -1
- package/dist/ui/static/atlassian-connectors.html +1 -1
- package/dist/ui/static/atlassian-connectors.txt +1 -1
- package/dist/ui/static/index.html +1 -1
- package/dist/ui/static/index.txt +1 -1
- package/dist/ui/static/launch/__next._full.txt +1 -1
- package/dist/ui/static/launch/__next._head.txt +1 -1
- package/dist/ui/static/launch/__next._index.txt +1 -1
- package/dist/ui/static/launch/__next._tree.txt +1 -1
- package/dist/ui/static/launch/__next.launch.__PAGE__.txt +1 -1
- package/dist/ui/static/launch/__next.launch.txt +1 -1
- package/dist/ui/static/launch.html +1 -1
- package/dist/ui/static/launch.txt +1 -1
- package/dist/ui/static/local-knowledge/__next._full.txt +1 -1
- package/dist/ui/static/local-knowledge/__next._head.txt +1 -1
- package/dist/ui/static/local-knowledge/__next._index.txt +1 -1
- package/dist/ui/static/local-knowledge/__next._tree.txt +1 -1
- package/dist/ui/static/local-knowledge/__next.local-knowledge.__PAGE__.txt +1 -1
- package/dist/ui/static/local-knowledge/__next.local-knowledge.txt +1 -1
- package/dist/ui/static/local-knowledge/capsule/__next._full.txt +1 -1
- package/dist/ui/static/local-knowledge/capsule/__next._head.txt +1 -1
- package/dist/ui/static/local-knowledge/capsule/__next._index.txt +1 -1
- package/dist/ui/static/local-knowledge/capsule/__next._tree.txt +1 -1
- package/dist/ui/static/local-knowledge/capsule/__next.local-knowledge.capsule.__PAGE__.txt +1 -1
- package/dist/ui/static/local-knowledge/capsule/__next.local-knowledge.capsule.txt +1 -1
- package/dist/ui/static/local-knowledge/capsule/__next.local-knowledge.txt +1 -1
- package/dist/ui/static/local-knowledge/capsule.html +1 -1
- package/dist/ui/static/local-knowledge/capsule.txt +1 -1
- package/dist/ui/static/local-knowledge.html +1 -1
- package/dist/ui/static/local-knowledge.txt +1 -1
- package/node_modules/@oscharko-dev/keiko-cli/dist/ui.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-cli/dist/ui.js +25 -6
- package/node_modules/@oscharko-dev/keiko-cli/package.json +13 -13
- package/node_modules/@oscharko-dev/keiko-connectors/package.json +3 -3
- package/node_modules/@oscharko-dev/keiko-contracts/dist/index.d.ts +2 -2
- package/node_modules/@oscharko-dev/keiko-contracts/dist/index.js +2 -2
- package/node_modules/@oscharko-dev/keiko-contracts/package.json +1 -1
- package/node_modules/@oscharko-dev/keiko-evaluations/package.json +9 -9
- package/node_modules/@oscharko-dev/keiko-evidence/dist/version.d.ts +1 -1
- package/node_modules/@oscharko-dev/keiko-evidence/dist/version.js +1 -1
- package/node_modules/@oscharko-dev/keiko-evidence/package.json +4 -4
- package/node_modules/@oscharko-dev/keiko-git/package.json +1 -1
- package/node_modules/@oscharko-dev/keiko-harness/dist/version.d.ts +1 -1
- package/node_modules/@oscharko-dev/keiko-harness/dist/version.js +1 -1
- package/node_modules/@oscharko-dev/keiko-harness/package.json +6 -6
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/connector-pod.d.ts +2 -0
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/connector-pod.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/connector-pod.js +1 -0
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/index.d.ts +1 -0
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/index.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/index.js +3 -0
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/indexing/embedding-batcher.d.ts +1 -0
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/indexing/embedding-batcher.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/indexing/embedding-batcher.js +341 -62
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/indexing/orchestrator.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/indexing/orchestrator.js +586 -132
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/indexing/types.d.ts +9 -0
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/indexing/types.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/knowledge-log.d.ts +20 -0
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/knowledge-log.d.ts.map +1 -0
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/knowledge-log.js +154 -0
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/manual-pod-refresh.d.ts +2 -0
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/manual-pod-refresh.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/manual-pod-refresh.js +1 -0
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/manual-pod.d.ts +2 -0
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/manual-pod.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/manual-pod.js +1 -0
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/repository-pod.d.ts +2 -0
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/repository-pod.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/repository-pod.js +1 -0
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/store.d.ts +2 -0
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/store.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/store.js +43 -1
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/version.d.ts +1 -1
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/version.js +1 -1
- package/node_modules/@oscharko-dev/keiko-local-knowledge/package.json +5 -5
- package/node_modules/@oscharko-dev/keiko-memory-capture/dist/version.d.ts +1 -1
- package/node_modules/@oscharko-dev/keiko-memory-capture/dist/version.js +1 -1
- package/node_modules/@oscharko-dev/keiko-memory-capture/package.json +3 -3
- package/node_modules/@oscharko-dev/keiko-memory-consolidation/dist/version.d.ts +1 -1
- package/node_modules/@oscharko-dev/keiko-memory-consolidation/dist/version.js +1 -1
- package/node_modules/@oscharko-dev/keiko-memory-consolidation/package.json +3 -3
- package/node_modules/@oscharko-dev/keiko-memory-governance/dist/version.d.ts +1 -1
- package/node_modules/@oscharko-dev/keiko-memory-governance/dist/version.js +1 -1
- package/node_modules/@oscharko-dev/keiko-memory-governance/package.json +3 -3
- package/node_modules/@oscharko-dev/keiko-memory-retrieval/dist/version.d.ts +1 -1
- package/node_modules/@oscharko-dev/keiko-memory-retrieval/dist/version.js +1 -1
- package/node_modules/@oscharko-dev/keiko-memory-retrieval/package.json +3 -3
- package/node_modules/@oscharko-dev/keiko-memory-vault/dist/version.d.ts +1 -1
- package/node_modules/@oscharko-dev/keiko-memory-vault/dist/version.js +1 -1
- package/node_modules/@oscharko-dev/keiko-memory-vault/package.json +3 -3
- package/node_modules/@oscharko-dev/keiko-model-gateway/dist/gateway.d.ts +15 -2
- package/node_modules/@oscharko-dev/keiko-model-gateway/dist/gateway.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-model-gateway/dist/gateway.js +147 -15
- package/node_modules/@oscharko-dev/keiko-model-gateway/dist/http.d.ts +3 -0
- package/node_modules/@oscharko-dev/keiko-model-gateway/dist/http.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-model-gateway/dist/http.js +188 -23
- package/node_modules/@oscharko-dev/keiko-model-gateway/dist/observability.d.ts +27 -0
- package/node_modules/@oscharko-dev/keiko-model-gateway/dist/observability.d.ts.map +1 -0
- package/node_modules/@oscharko-dev/keiko-model-gateway/dist/observability.js +180 -0
- package/node_modules/@oscharko-dev/keiko-model-gateway/dist/openai-embedding-adapter.d.ts +5 -0
- package/node_modules/@oscharko-dev/keiko-model-gateway/dist/openai-embedding-adapter.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-model-gateway/dist/openai-embedding-adapter.js +286 -68
- package/node_modules/@oscharko-dev/keiko-model-gateway/dist/resilience.d.ts +18 -5
- package/node_modules/@oscharko-dev/keiko-model-gateway/dist/resilience.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-model-gateway/dist/resilience.js +141 -26
- package/node_modules/@oscharko-dev/keiko-model-gateway/dist/version.d.ts +1 -1
- package/node_modules/@oscharko-dev/keiko-model-gateway/dist/version.js +1 -1
- package/node_modules/@oscharko-dev/keiko-model-gateway/package.json +3 -3
- package/node_modules/@oscharko-dev/keiko-quality-intelligence/package.json +3 -3
- package/node_modules/@oscharko-dev/keiko-sandbox/package.json +2 -2
- package/node_modules/@oscharko-dev/keiko-sdk/package.json +9 -9
- package/node_modules/@oscharko-dev/keiko-security/dist/version.d.ts +1 -1
- package/node_modules/@oscharko-dev/keiko-security/dist/version.js +1 -1
- package/node_modules/@oscharko-dev/keiko-security/package.json +2 -2
- package/node_modules/@oscharko-dev/keiko-server/dist/atlassian/syncService.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-server/dist/atlassian/syncService.js +4 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/bounded-request-body.d.ts +1 -1
- package/node_modules/@oscharko-dev/keiko-server/dist/bounded-request-body.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-server/dist/bounded-request-body.js +51 -3
- package/node_modules/@oscharko-dev/keiko-server/dist/diagnostics-log.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-server/dist/diagnostics-log.js +77 -3
- package/node_modules/@oscharko-dev/keiko-server/dist/gateway-instance-cache.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-server/dist/gateway-instance-cache.js +71 -11
- package/node_modules/@oscharko-dev/keiko-server/dist/gateway-setup.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-server/dist/gateway-setup.js +9 -1
- package/node_modules/@oscharko-dev/keiko-server/dist/grounded-qa.js +1 -1
- package/node_modules/@oscharko-dev/keiko-server/dist/grounded-rerank-facade.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-server/dist/grounded-rerank-facade.js +37 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/index.d.ts +1 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/index.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-server/dist/index.js +7 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/local-knowledge-handlers.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-server/dist/local-knowledge-handlers.js +158 -36
- package/node_modules/@oscharko-dev/keiko-server/dist/local-knowledge-remediation-store.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-server/dist/local-knowledge-remediation-store.js +9 -1
- package/node_modules/@oscharko-dev/keiko-server/dist/local-knowledge-remediation.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-server/dist/local-knowledge-remediation.js +10 -5
- package/node_modules/@oscharko-dev/keiko-server/dist/local-knowledge-store-open.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-server/dist/local-knowledge-store-open.js +8 -2
- package/node_modules/@oscharko-dev/keiko-server/dist/manual/manual-pod-service.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-server/dist/manual/manual-pod-service.js +3 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/memory-embedding.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-server/dist/memory-embedding.js +135 -36
- package/node_modules/@oscharko-dev/keiko-server/dist/observability/index.d.ts +3 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/observability/index.d.ts.map +1 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/observability/index.js +9 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/observability/log-level.d.ts +13 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/observability/log-level.d.ts.map +1 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/observability/log-level.js +57 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/observability/log-redaction.d.ts +20 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/observability/log-redaction.d.ts.map +1 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/observability/log-redaction.js +499 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/observability/route-template.d.ts +4 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/observability/route-template.d.ts.map +1 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/observability/route-template.js +340 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/observability/server-log.d.ts +49 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/observability/server-log.d.ts.map +1 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/observability/server-log.js +449 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/observability/server-logger.d.ts +36 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/observability/server-logger.d.ts.map +1 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/observability/server-logger.js +190 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/process-log-sink.d.ts +11 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/process-log-sink.d.ts.map +1 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/process-log-sink.js +48 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/server.d.ts +2 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/server.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-server/dist/server.js +22 -4
- package/node_modules/@oscharko-dev/keiko-server/dist/store-handlers.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-server/dist/store-handlers.js +7 -2
- package/node_modules/@oscharko-dev/keiko-server/package.json +21 -21
- package/node_modules/@oscharko-dev/keiko-tools/dist/version.d.ts +1 -1
- package/node_modules/@oscharko-dev/keiko-tools/dist/version.js +1 -1
- package/node_modules/@oscharko-dev/keiko-tools/package.json +5 -5
- package/node_modules/@oscharko-dev/keiko-verification/package.json +5 -5
- package/node_modules/@oscharko-dev/keiko-workflows/package.json +10 -10
- package/node_modules/@oscharko-dev/keiko-workspace/dist/version.d.ts +1 -1
- package/node_modules/@oscharko-dev/keiko-workspace/dist/version.js +1 -1
- package/node_modules/@oscharko-dev/keiko-workspace/package.json +3 -3
- package/package.json +25 -25
- package/release-impact.catalog.json +55 -0
- package/vendor/{oscharko-dev-keiko-cli-0.3.14.tgz → oscharko-dev-keiko-cli-0.3.15.tgz} +0 -0
- package/vendor/{oscharko-dev-keiko-connectors-0.3.14.tgz → oscharko-dev-keiko-connectors-0.3.15.tgz} +0 -0
- package/vendor/{oscharko-dev-keiko-contracts-0.3.14.tgz → oscharko-dev-keiko-contracts-0.3.15.tgz} +0 -0
- package/vendor/{oscharko-dev-keiko-evaluations-0.3.14.tgz → oscharko-dev-keiko-evaluations-0.3.15.tgz} +0 -0
- package/vendor/{oscharko-dev-keiko-evidence-0.3.14.tgz → oscharko-dev-keiko-evidence-0.3.15.tgz} +0 -0
- package/vendor/{oscharko-dev-keiko-git-0.3.14.tgz → oscharko-dev-keiko-git-0.3.15.tgz} +0 -0
- package/vendor/{oscharko-dev-keiko-harness-0.3.14.tgz → oscharko-dev-keiko-harness-0.3.15.tgz} +0 -0
- package/vendor/oscharko-dev-keiko-local-knowledge-0.3.15.tgz +0 -0
- package/vendor/{oscharko-dev-keiko-memory-capture-0.3.14.tgz → oscharko-dev-keiko-memory-capture-0.3.15.tgz} +0 -0
- package/vendor/{oscharko-dev-keiko-memory-consolidation-0.3.14.tgz → oscharko-dev-keiko-memory-consolidation-0.3.15.tgz} +0 -0
- package/vendor/{oscharko-dev-keiko-memory-governance-0.3.14.tgz → oscharko-dev-keiko-memory-governance-0.3.15.tgz} +0 -0
- package/vendor/{oscharko-dev-keiko-memory-retrieval-0.3.14.tgz → oscharko-dev-keiko-memory-retrieval-0.3.15.tgz} +0 -0
- package/vendor/{oscharko-dev-keiko-memory-vault-0.3.14.tgz → oscharko-dev-keiko-memory-vault-0.3.15.tgz} +0 -0
- package/vendor/oscharko-dev-keiko-model-gateway-0.3.15.tgz +0 -0
- package/vendor/{oscharko-dev-keiko-quality-intelligence-0.3.14.tgz → oscharko-dev-keiko-quality-intelligence-0.3.15.tgz} +0 -0
- package/vendor/{oscharko-dev-keiko-sandbox-0.3.14.tgz → oscharko-dev-keiko-sandbox-0.3.15.tgz} +0 -0
- package/vendor/oscharko-dev-keiko-sdk-0.3.15.tgz +0 -0
- package/vendor/{oscharko-dev-keiko-security-0.3.14.tgz → oscharko-dev-keiko-security-0.3.15.tgz} +0 -0
- package/vendor/oscharko-dev-keiko-server-0.3.15.tgz +0 -0
- package/vendor/{oscharko-dev-keiko-tools-0.3.14.tgz → oscharko-dev-keiko-tools-0.3.15.tgz} +0 -0
- package/vendor/{oscharko-dev-keiko-verification-0.3.14.tgz → oscharko-dev-keiko-verification-0.3.15.tgz} +0 -0
- package/vendor/{oscharko-dev-keiko-workflows-0.3.14.tgz → oscharko-dev-keiko-workflows-0.3.15.tgz} +0 -0
- package/vendor/{oscharko-dev-keiko-workspace-0.3.14.tgz → oscharko-dev-keiko-workspace-0.3.15.tgz} +0 -0
- package/vendor/oscharko-dev-keiko-local-knowledge-0.3.14.tgz +0 -0
- package/vendor/oscharko-dev-keiko-model-gateway-0.3.14.tgz +0 -0
- package/vendor/oscharko-dev-keiko-sdk-0.3.14.tgz +0 -0
- package/vendor/oscharko-dev-keiko-server-0.3.14.tgz +0 -0
- /package/dist/ui/static/_next/static/{y54UtqSyUZo4acoo_2Vb4 → 4j3JDaPBERRu_ZIgeR1VQ}/_buildManifest.js +0 -0
- /package/dist/ui/static/_next/static/{y54UtqSyUZo4acoo_2Vb4 → 4j3JDaPBERRu_ZIgeR1VQ}/_clientMiddlewareManifest.js +0 -0
- /package/dist/ui/static/_next/static/{y54UtqSyUZo4acoo_2Vb4 → 4j3JDaPBERRu_ZIgeR1VQ}/_ssgManifest.js +0 -0
|
@@ -24,7 +24,7 @@
|
|
|
24
24
|
// handleFileExtracted) so chunk-and-embed re-runs even for unchanged file hashes.
|
|
25
25
|
// Recovery mode (partial vector coverage, non-force): re-embeds using existing chunks
|
|
26
26
|
// only — the chunker runs with force=false so it reuses the already-correct chunk rows.
|
|
27
|
-
import { randomUUID } from "node:crypto";
|
|
27
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
28
28
|
import { checkpointCompatibility, DEFAULT_LARGE_DOCUMENT_RESOURCE_POLICY, LARGE_DOCUMENT_DIAGNOSTIC_CODES, largeDocumentPolicyFingerprint, } from "@oscharko-dev/keiko-contracts";
|
|
29
29
|
import { assertCompatibleEmbeddingIdentity, verifyEmbeddingCapability, } from "@oscharko-dev/keiko-model-gateway";
|
|
30
30
|
import { chunkDocument } from "../chunking/chunker-runner.js";
|
|
@@ -40,7 +40,8 @@ import { BoundedIndexingCancelledError, BoundedIndexingPolicyError, chunkDocumen
|
|
|
40
40
|
import { resolveCapsuleModelUsePolicy } from "../model-use-policy.js";
|
|
41
41
|
import { selectExtractionCheckpoint, upsertExtractionCheckpoint } from "./checkpoint-persist.js";
|
|
42
42
|
import { finalizeJobRow, isJobCancellationRequested, insertJobRow, updateJobCounters, } from "./job-persist.js";
|
|
43
|
-
import { embedChunkBatch } from "./embedding-batcher.js";
|
|
43
|
+
import { embedChunkBatch, embeddingEndpointHost } from "./embedding-batcher.js";
|
|
44
|
+
import { emitKnowledgeLogEvent, knowledgeErrorKind, startKnowledgeLogTimer, } from "../knowledge-log.js";
|
|
44
45
|
import { countVectorsForCapsule, countVectorsForDocument, deleteVectorsForDocument, invalidateVectorIndexStateForCapsules, selectChunksForDocument, } from "./vector-persist.js";
|
|
45
46
|
import { countLexicalRowsForDocument, deleteLexicalRowsForDocument, replaceLexicalRowsForDocument, } from "./lexical-index-persist.js";
|
|
46
47
|
import { DEFAULT_INDEXING_BATCH_SIZE, DEFAULT_INDEXING_CONCURRENCY, IndexingError, } from "./types.js";
|
|
@@ -141,6 +142,74 @@ function emitProgress(options, event) {
|
|
|
141
142
|
// intentionally swallowed — progress sinks must not affect run correctness
|
|
142
143
|
}
|
|
143
144
|
}
|
|
145
|
+
// ─── Activity log ─────────────────────────────────────────────────────────────
|
|
146
|
+
// WHY THESE LINES EXIST ON TOP OF THE EVENT STREAM
|
|
147
|
+
//
|
|
148
|
+
// `IndexingEvent` reports STATE, and only to a consumer that is actively driving the async
|
|
149
|
+
// iterator. The field incident this instrumentation was written for is a run that produced no
|
|
150
|
+
// state change at all — "0 of 1 documents, 0 of 36 vectors" for six minutes, then an operator
|
|
151
|
+
// cancellation. There was nothing to report, so nothing was reported, and four releases went by
|
|
152
|
+
// guessing. These lines are the run's SPINE: one per lifecycle transition, at `info`, so an
|
|
153
|
+
// operator who opens `server.log` at the default level sees which step a stuck run reached and
|
|
154
|
+
// never started the next one — without attaching a consumer to anything.
|
|
155
|
+
//
|
|
156
|
+
// CORRELATION. Concurrency is up to 4, so several documents and several embedding flushes are in
|
|
157
|
+
// flight at once and a line without an owner cannot be attributed to the work that produced it.
|
|
158
|
+
// Every line carries the run's `IndexingLogContext`: the job uuid verbatim in `correlationId`,
|
|
159
|
+
// and the capsule (always) and document (where known) as DIGESTS — see the type's own note for
|
|
160
|
+
// why the raw ids are not writable.
|
|
161
|
+
const LOG_DIGEST_LENGTH = 16;
|
|
162
|
+
// A truncated sha-256. 16 hex characters is 64 bits: collision-free across every capsule and
|
|
163
|
+
// document a single log file can hold, short enough to read at the start of a line, and — being
|
|
164
|
+
// a one-way digest of the whole value — carries nothing back about the name it stands for.
|
|
165
|
+
function logDigest(value) {
|
|
166
|
+
return createHash("sha256").update(value).digest("hex").slice(0, LOG_DIGEST_LENGTH);
|
|
167
|
+
}
|
|
168
|
+
// `scheme://host` for the configured embedding gateway, omitted when the endpoint does not
|
|
169
|
+
// parse. Never the path or the query — see `embeddingEndpointHost`.
|
|
170
|
+
function endpointHostExtra(state) {
|
|
171
|
+
const host = embeddingEndpointHost(state.options.embeddingAdapter.endpoint);
|
|
172
|
+
return host === undefined ? {} : { endpointHost: host };
|
|
173
|
+
}
|
|
174
|
+
function documentLogContext(state, documentId) {
|
|
175
|
+
return { ...state.logContext, documentIdDigest: logDigest(String(documentId)) };
|
|
176
|
+
}
|
|
177
|
+
// Routed through `emitKnowledgeLogEvent` for the same reason `emitProgress` above wraps the
|
|
178
|
+
// progress callback: the sink is caller-supplied code, and a run's correctness may not depend on
|
|
179
|
+
// it. Without the guard a throwing sink would abort whichever step happened to be logging —
|
|
180
|
+
// mid-document, inside a retry ladder, or in the failure path that is about to report the real
|
|
181
|
+
// cause — and a logging defect would be indistinguishable from an indexing failure. The seam also
|
|
182
|
+
// keeps a broken sink from going unnoticed; see `knowledge-log.ts`.
|
|
183
|
+
function writeKnowledgeLog(state, context, event) {
|
|
184
|
+
emitKnowledgeLogEvent(state.options.logSink, {
|
|
185
|
+
...event,
|
|
186
|
+
correlationId: context.jobId,
|
|
187
|
+
extra: {
|
|
188
|
+
capsuleIdDigest: context.capsuleIdDigest,
|
|
189
|
+
...(context.documentIdDigest !== undefined
|
|
190
|
+
? { documentIdDigest: context.documentIdDigest }
|
|
191
|
+
: {}),
|
|
192
|
+
...event.extra,
|
|
193
|
+
},
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
// Job-scoped lifecycle line.
|
|
197
|
+
function logIndexing(state, event) {
|
|
198
|
+
writeKnowledgeLog(state, state.logContext, { ...event, category: "indexing" });
|
|
199
|
+
}
|
|
200
|
+
// Job-scoped line about the embedding gateway rather than the corpus. The category has to match
|
|
201
|
+
// the op prefix: an `embedding.*` op filed under `indexing` is invisible to the grep an operator
|
|
202
|
+
// runs when the gateway is the suspect.
|
|
203
|
+
function logEmbeddingRun(state, event) {
|
|
204
|
+
writeKnowledgeLog(state, state.logContext, { ...event, category: "embedding" });
|
|
205
|
+
}
|
|
206
|
+
// Document-scoped lifecycle line — same shape, plus the document digest.
|
|
207
|
+
function logDocument(state, documentId, event) {
|
|
208
|
+
writeKnowledgeLog(state, documentLogContext(state, documentId), {
|
|
209
|
+
...event,
|
|
210
|
+
category: "indexing",
|
|
211
|
+
});
|
|
212
|
+
}
|
|
144
213
|
function clearDocumentArtifacts(state, documentId, options) {
|
|
145
214
|
deleteVectorsForDocument(state.options.store._internal.db, state.capsule.id, documentId);
|
|
146
215
|
if (options.deleteChunks) {
|
|
@@ -177,12 +246,7 @@ function eligibleForRestoreSnapshot(db, capsuleId, documentId) {
|
|
|
177
246
|
return false;
|
|
178
247
|
return countVectorsForDocument(db, capsuleId, documentId) > 0;
|
|
179
248
|
}
|
|
180
|
-
function captureRestoreSnapshotIfEligible(state,
|
|
181
|
-
const documentId = documentIdFor({
|
|
182
|
-
capsuleId: state.capsule.id,
|
|
183
|
-
sourceId: source.id,
|
|
184
|
-
relativePath,
|
|
185
|
-
});
|
|
249
|
+
function captureRestoreSnapshotIfEligible(state, documentId) {
|
|
186
250
|
const db = state.options.store._internal.db;
|
|
187
251
|
if (!eligibleForRestoreSnapshot(db, state.capsule.id, documentId))
|
|
188
252
|
return;
|
|
@@ -510,7 +574,20 @@ async function prepareChunksToEmbedSafely(state, documentId, sourceText) {
|
|
|
510
574
|
try {
|
|
511
575
|
return await prepareChunksToEmbed(state, documentId, sourceText);
|
|
512
576
|
}
|
|
513
|
-
catch {
|
|
577
|
+
catch (cause) {
|
|
578
|
+
// The caller converts `null` into a CONTEXTUAL_RETRIEVAL_FAILED document error, which tells
|
|
579
|
+
// an operator that a document failed but not which stage threw. The error KIND is the one
|
|
580
|
+
// fact that separates a chat-gateway outage from a chunker defect, and it is lost here
|
|
581
|
+
// unless this line records it. The message is never read — it can carry document text, and
|
|
582
|
+
// neither is the raw document id, which IS the document's relative path (`doc:<capsule>:
|
|
583
|
+
// <source>:<relativePath>`): the digest correlates it to every other line of this document's
|
|
584
|
+
// work without writing a customer file name into the log.
|
|
585
|
+
logDocument(state, documentId, {
|
|
586
|
+
level: "warn",
|
|
587
|
+
op: "indexing.chunking.failed",
|
|
588
|
+
errorKind: knowledgeErrorKind(cause),
|
|
589
|
+
extra: { sourceTextLength: sourceText.length, lane: "standard" },
|
|
590
|
+
});
|
|
514
591
|
return null;
|
|
515
592
|
}
|
|
516
593
|
}
|
|
@@ -520,7 +597,7 @@ function lastChunkIdOfBatch(vectors) {
|
|
|
520
597
|
const last = vectors.at(-1);
|
|
521
598
|
return last === undefined ? null : last.chunkId;
|
|
522
599
|
}
|
|
523
|
-
async function embedOneChunkBatch(state, batch) {
|
|
600
|
+
async function embedOneChunkBatch(state, documentId, batch) {
|
|
524
601
|
return embedChunkBatch(batch, {
|
|
525
602
|
adapter: state.options.embeddingAdapter,
|
|
526
603
|
store: state.options.store,
|
|
@@ -531,16 +608,22 @@ async function embedOneChunkBatch(state, batch) {
|
|
|
531
608
|
now: state.now,
|
|
532
609
|
idSource: state.idSource,
|
|
533
610
|
tokenizer: state.tokenizer,
|
|
611
|
+
// The batcher's lines are the ones an operator reads during a stall, and with concurrency 4
|
|
612
|
+
// they interleave across documents — so they carry the same correlation context as the
|
|
613
|
+
// orchestrator's own, down to the document.
|
|
614
|
+
...(state.options.logSink !== undefined
|
|
615
|
+
? { logSink: state.options.logSink, logContext: documentLogContext(state, documentId) }
|
|
616
|
+
: {}),
|
|
534
617
|
});
|
|
535
618
|
}
|
|
536
|
-
async function embedChunkBatches(state, batches) {
|
|
619
|
+
async function embedChunkBatches(state, documentId, batches) {
|
|
537
620
|
const errors = [];
|
|
538
621
|
let vectorCount = 0;
|
|
539
622
|
let lastChunkId = null;
|
|
540
623
|
for (const batch of batches) {
|
|
541
624
|
if (cancellationRequested(state))
|
|
542
625
|
break;
|
|
543
|
-
const result = await embedOneChunkBatch(state, batch);
|
|
626
|
+
const result = await embedOneChunkBatch(state, documentId, batch);
|
|
544
627
|
vectorCount += result.vectors.length;
|
|
545
628
|
errors.push(...result.errors);
|
|
546
629
|
const batchLastChunkId = lastChunkIdOfBatch(result.vectors);
|
|
@@ -570,70 +653,111 @@ async function embedDocumentChunks(state, documentId, source, relativePath) {
|
|
|
570
653
|
return { vectorCount: 0, errors: [], lastChunkId: null };
|
|
571
654
|
}
|
|
572
655
|
const batches = sliceIntoBatches(chunks, state.batchSize);
|
|
573
|
-
|
|
656
|
+
// The flush plan for this document: how many chunks became how many batches at which cap. It
|
|
657
|
+
// is the last line before the first outbound call, so a run that stalls in embedding stops
|
|
658
|
+
// exactly here — with the count of work it was about to issue.
|
|
659
|
+
logDocument(state, documentId, {
|
|
660
|
+
level: "info",
|
|
661
|
+
op: "indexing.document.embedding-started",
|
|
662
|
+
extra: { chunkCount: chunks.length, batchCount: batches.length, batchSize: state.batchSize },
|
|
663
|
+
});
|
|
664
|
+
return embedChunkBatches(state, documentId, batches);
|
|
574
665
|
}
|
|
575
666
|
// ─── Document handlers ────────────────────────────────────────────────────────
|
|
576
667
|
function handleExtractionSkipped(state, result) {
|
|
577
668
|
state.skippedDocuments += 1;
|
|
669
|
+
const documentId = result.outcome.kind === "skipped" ? result.outcome.document.id : "";
|
|
670
|
+
logDocument(state, documentId, {
|
|
671
|
+
level: "info",
|
|
672
|
+
op: "indexing.document.skipped",
|
|
673
|
+
extra: { reason: "unchanged", skippedDocuments: state.skippedDocuments },
|
|
674
|
+
});
|
|
578
675
|
return {
|
|
579
676
|
kind: "document-skipped",
|
|
580
677
|
jobId: state.jobId,
|
|
581
678
|
capsuleId: state.capsule.id,
|
|
582
679
|
sourceId: result.sourceId,
|
|
583
|
-
documentId
|
|
680
|
+
documentId,
|
|
584
681
|
reason: "unchanged",
|
|
585
682
|
};
|
|
586
683
|
}
|
|
587
684
|
// GRD-010: transient IO failure codes that must NOT destroy a previously-good index on an
|
|
588
685
|
// incremental refresh. Mirrors the gate in discovery/extract.ts buildFailureResult.
|
|
589
686
|
const TRANSIENT_DISCOVERY_CODES = new Set(["READ_FAILED", "STAT_FAILED"]);
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
687
|
+
// The extraction lane does NOT reach `appendDocumentFailure` (see the note there), so until this
|
|
688
|
+
// line existed a READ_FAILED, a STAT_FAILED, a PATH_ESCAPE or any parse failure produced a
|
|
689
|
+
// `DISCOVERY_FAILED:<code>` event for a consumer driving the iterator and nothing at all in the
|
|
690
|
+
// file an operator opens. `errorKind` carries the discovery code — the part that says WHY, and
|
|
691
|
+
// the part that decides the repair: an unreadable mount, an unstattable entry and a parser that
|
|
692
|
+
// rejected the bytes are three different problems. The message is never written; it quotes the
|
|
693
|
+
// path, or the fragment of content that failed to parse.
|
|
694
|
+
function logExtractionFailed(state, documentId, discoveryCode) {
|
|
695
|
+
logDocument(state, documentId, {
|
|
696
|
+
level: "warn",
|
|
697
|
+
op: "indexing.document.extraction-failed",
|
|
698
|
+
errorKind: discoveryCode,
|
|
699
|
+
extra: { failedDocuments: state.failedDocuments },
|
|
700
|
+
});
|
|
701
|
+
}
|
|
702
|
+
// The GRD-010 downgrade, made visible. A transient re-read failure on a document whose previous
|
|
703
|
+
// chunks survived is reported to the consumer as `document-skipped` with reason "unchanged" —
|
|
704
|
+
// byte-identical to what a genuinely unchanged document produces. The flattening is deliberate
|
|
705
|
+
// (the retrievable content really did survive), but it hides that this document was NOT
|
|
706
|
+
// refreshed this run, which is exactly the state behind "the pod keeps answering from the old
|
|
707
|
+
// version of the file". At `warn`, with the discovery code that caused the downgrade.
|
|
708
|
+
function transientRereadSkip(state, result, documentId, discoveryCode, preservedChunkCount) {
|
|
709
|
+
state.skippedDocuments += 1;
|
|
710
|
+
logDocument(state, documentId, {
|
|
711
|
+
level: "warn",
|
|
712
|
+
op: "indexing.document.skipped",
|
|
713
|
+
errorKind: discoveryCode,
|
|
714
|
+
extra: {
|
|
715
|
+
reason: "transient-read-failure",
|
|
716
|
+
preservedChunkCount,
|
|
717
|
+
skippedDocuments: state.skippedDocuments,
|
|
718
|
+
},
|
|
719
|
+
});
|
|
720
|
+
return {
|
|
721
|
+
kind: "document-skipped",
|
|
722
|
+
jobId: state.jobId,
|
|
723
|
+
capsuleId: state.capsule.id,
|
|
724
|
+
sourceId: result.sourceId,
|
|
725
|
+
documentId,
|
|
726
|
+
reason: "unchanged",
|
|
727
|
+
};
|
|
728
|
+
}
|
|
729
|
+
function failExtractedDocument(state, result, documentId, discoveryCode, message) {
|
|
625
730
|
state.failedDocuments += 1;
|
|
626
|
-
|
|
731
|
+
logExtractionFailed(state, documentId, discoveryCode);
|
|
732
|
+
clearDocumentArtifacts(state, documentId, { deleteChunks: true });
|
|
733
|
+
markDocumentFailed(state, documentId);
|
|
734
|
+
const error = { code: `DISCOVERY_FAILED:${discoveryCode}`, message };
|
|
627
735
|
state.lastError = error;
|
|
628
736
|
return {
|
|
629
737
|
kind: "document-failed",
|
|
630
738
|
jobId: state.jobId,
|
|
631
739
|
capsuleId: state.capsule.id,
|
|
632
740
|
sourceId: result.sourceId,
|
|
741
|
+
documentId,
|
|
633
742
|
relativePath: result.relativePath,
|
|
634
743
|
error,
|
|
635
744
|
};
|
|
636
745
|
}
|
|
746
|
+
function handleExtractionFailed(state, result, outcome) {
|
|
747
|
+
const documentId = outcome.document.id;
|
|
748
|
+
const errCode = outcome.error.code;
|
|
749
|
+
// GRD-010: a transient re-read failure on a document that still has a prior good index
|
|
750
|
+
// (extract.ts preserved its chunks/vectors) is reported as a non-destructive skip, NOT a
|
|
751
|
+
// failure — the retrievable content survives until a successful re-extraction. The chunk
|
|
752
|
+
// count is only read for a transient code, exactly as before.
|
|
753
|
+
const preservedChunks = TRANSIENT_DISCOVERY_CODES.has(errCode)
|
|
754
|
+
? countChunksForDocument(state.options.store._internal.db, state.capsule.id, documentId)
|
|
755
|
+
: 0;
|
|
756
|
+
if (preservedChunks > 0) {
|
|
757
|
+
return transientRereadSkip(state, result, documentId, errCode, preservedChunks);
|
|
758
|
+
}
|
|
759
|
+
return failExtractedDocument(state, result, documentId, errCode, outcome.error.message);
|
|
760
|
+
}
|
|
637
761
|
function resolveChunkCount(state, documentId, skippedExisting, freshChunkIds) {
|
|
638
762
|
if (!skippedExisting)
|
|
639
763
|
return freshChunkIds.length;
|
|
@@ -677,6 +801,19 @@ function chunkPersistedDocument(state, result) {
|
|
|
677
801
|
};
|
|
678
802
|
}
|
|
679
803
|
function chunkedDocumentEvents(state, sourceId, documentId, relativePath, chunkCount) {
|
|
804
|
+
// The per-document half of the spine. `chunkCount` is the denominator of the "0 of 36 vectors"
|
|
805
|
+
// the operator sees: with this line an operator can say the chunker produced 36 and embedding
|
|
806
|
+
// never returned, which is a different bug from the chunker producing nothing. The relative
|
|
807
|
+
// path is deliberately absent — it is a customer file name; the document digest identifies it.
|
|
808
|
+
logDocument(state, documentId, {
|
|
809
|
+
level: "info",
|
|
810
|
+
op: "indexing.document.extracted",
|
|
811
|
+
});
|
|
812
|
+
logDocument(state, documentId, {
|
|
813
|
+
level: "info",
|
|
814
|
+
op: "indexing.document.chunked",
|
|
815
|
+
extra: { chunkCount },
|
|
816
|
+
});
|
|
680
817
|
return [
|
|
681
818
|
{
|
|
682
819
|
kind: "document-extracted",
|
|
@@ -759,32 +896,25 @@ function tryChunkDocument(state, result, documentId) {
|
|
|
759
896
|
try {
|
|
760
897
|
return { chunked: chunkPersistedDocument(state, result) };
|
|
761
898
|
}
|
|
762
|
-
catch {
|
|
899
|
+
catch (cause) {
|
|
763
900
|
if (cancellationRequested(state)) {
|
|
764
901
|
clearDocumentArtifacts(state, documentId, { deleteChunks: true });
|
|
765
902
|
return { events: [] };
|
|
766
903
|
}
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
sourceId: result.sourceId,
|
|
782
|
-
documentId,
|
|
783
|
-
relativePath: result.relativePath,
|
|
784
|
-
error,
|
|
785
|
-
},
|
|
786
|
-
],
|
|
787
|
-
};
|
|
904
|
+
// Same class as the contextual lane above and the bounded lane below: the document error is
|
|
905
|
+
// flattened to CHUNKING_FAILED, so the real cause exists nowhere else. One shared op name
|
|
906
|
+
// across all three lanes, distinguished by `lane`, so an operator greps once.
|
|
907
|
+
logDocument(state, documentId, {
|
|
908
|
+
level: "warn",
|
|
909
|
+
op: "indexing.chunking.failed",
|
|
910
|
+
errorKind: knowledgeErrorKind(cause),
|
|
911
|
+
extra: { lane: "standard-chunker" },
|
|
912
|
+
});
|
|
913
|
+
// Routed through the shared failure sink rather than a hand-built copy of it: this branch
|
|
914
|
+
// used to duplicate the counter bump, the artifact cleanup, the status write and the event,
|
|
915
|
+
// which is how it ended up as the one document failure with no `indexing.document.failed`
|
|
916
|
+
// line. One owner, one shape, every lane.
|
|
917
|
+
return appendDocumentFailure(state, [], result.sourceId, documentId, result.relativePath, { code: "CHUNKING_FAILED", message: "document chunking failed" }, { deleteChunks: true });
|
|
788
918
|
}
|
|
789
919
|
}
|
|
790
920
|
function appendDocumentFailure(state, events, sourceId, documentId, relativePath, error, options) {
|
|
@@ -792,6 +922,22 @@ function appendDocumentFailure(state, events, sourceId, documentId, relativePath
|
|
|
792
922
|
clearDocumentArtifacts(state, documentId, options);
|
|
793
923
|
markDocumentFailed(state, documentId);
|
|
794
924
|
state.lastError = error;
|
|
925
|
+
// Every POST-EXTRACTION lane funnels through here — standard, bounded, identity gate and
|
|
926
|
+
// chunker alike — so this is the single site that records their failures. It is NOT every
|
|
927
|
+
// failing lane: a document that never got past discovery/extraction fails inside
|
|
928
|
+
// `handleExtractionFailed`, which owns its own `indexing.document.extraction-failed` line for
|
|
929
|
+
// exactly that reason. Claiming otherwise here is what left that lane silent through four
|
|
930
|
+
// releases of field guesswork. The error CODE is written, never the message: a document
|
|
931
|
+
// error's message can quote the content that failed to parse.
|
|
932
|
+
logDocument(state, documentId, {
|
|
933
|
+
level: "warn",
|
|
934
|
+
op: "indexing.document.failed",
|
|
935
|
+
errorKind: error.code,
|
|
936
|
+
extra: {
|
|
937
|
+
transient: error.transient === true,
|
|
938
|
+
consecutiveTransientEmbedFailures: state.consecutiveTransientEmbedFailures,
|
|
939
|
+
},
|
|
940
|
+
});
|
|
795
941
|
events.push({
|
|
796
942
|
kind: "document-failed",
|
|
797
943
|
jobId: state.jobId,
|
|
@@ -808,6 +954,15 @@ function completeEmbeddedDocument(state, events, sourceId, documentId, embedResu
|
|
|
808
954
|
state.vectorsPersisted += embedResult.vectorCount;
|
|
809
955
|
if (embedResult.lastChunkId !== null)
|
|
810
956
|
state.lastResumeToken = embedResult.lastChunkId;
|
|
957
|
+
logDocument(state, documentId, {
|
|
958
|
+
level: "info",
|
|
959
|
+
op: "indexing.document.embedded",
|
|
960
|
+
extra: {
|
|
961
|
+
vectorCount: embedResult.vectorCount,
|
|
962
|
+
vectorsPersistedSoFar: state.vectorsPersisted,
|
|
963
|
+
processedDocuments: state.processedDocuments,
|
|
964
|
+
},
|
|
965
|
+
});
|
|
811
966
|
events.push({
|
|
812
967
|
kind: "document-embedded",
|
|
813
968
|
jobId: state.jobId,
|
|
@@ -984,8 +1139,27 @@ function prepareBoundedChunks(state, result, documentId, checkpoint, fingerprint
|
|
|
984
1139
|
}
|
|
985
1140
|
return countChunksForDocument(db, state.capsule.id, documentId);
|
|
986
1141
|
}
|
|
1142
|
+
// The bounded large-document lane had the same hole the standard lane did: every non-policy cause
|
|
1143
|
+
// is relabelled `CHUNKING_FAILED` with a fixed message, so an out-of-memory page window, a corrupt
|
|
1144
|
+
// checkpoint and a real chunker defect are indistinguishable downstream. It gets the identical
|
|
1145
|
+
// line, under the identical op name, tagged with its lane — including the cancellation branch,
|
|
1146
|
+
// which otherwise returns zero events and leaves an operator with a document that simply stopped
|
|
1147
|
+
// existing in the run.
|
|
1148
|
+
function logBoundedChunkFailure(state, documentId, cause) {
|
|
1149
|
+
logDocument(state, documentId, {
|
|
1150
|
+
level: "warn",
|
|
1151
|
+
op: "indexing.chunking.failed",
|
|
1152
|
+
errorKind: knowledgeErrorKind(cause),
|
|
1153
|
+
extra: {
|
|
1154
|
+
lane: "bounded",
|
|
1155
|
+
cancelled: cause instanceof BoundedIndexingCancelledError || cancellationRequested(state),
|
|
1156
|
+
policyRejection: cause instanceof BoundedIndexingPolicyError,
|
|
1157
|
+
},
|
|
1158
|
+
});
|
|
1159
|
+
}
|
|
987
1160
|
function boundedChunkPreparationFailure(state, result, documentId, checkpoint, fingerprint, cause) {
|
|
988
1161
|
const db = state.options.store._internal.db;
|
|
1162
|
+
logBoundedChunkFailure(state, documentId, cause);
|
|
989
1163
|
if (cause instanceof BoundedIndexingCancelledError || cancellationRequested(state)) {
|
|
990
1164
|
updateDocumentStatusRow(db, state.capsule.id, documentId, "pending");
|
|
991
1165
|
writeBoundedCheckpoint({
|
|
@@ -1107,6 +1281,21 @@ async function* handleBoundedDocument(state, result, documentId, checkpoint) {
|
|
|
1107
1281
|
function* handleUnsupportedDocument(state, result, documentId) {
|
|
1108
1282
|
clearDocumentArtifacts(state, documentId, { deleteChunks: true });
|
|
1109
1283
|
state.skippedDocuments += 1;
|
|
1284
|
+
// This pair — extracted, then immediately skipped — is a whole document that will never
|
|
1285
|
+
// produce a vector, and it wrote nothing. An operator reading "0 of 1 documents" against a
|
|
1286
|
+
// scanned PDF or an image saw a run that looked identical to one wedged in the gateway. The
|
|
1287
|
+
// `extracted` line keeps this lane countable next to the chunked lane's own; `documentStatus`
|
|
1288
|
+
// is the closed enum the extractor assigned, never a file name.
|
|
1289
|
+
logDocument(state, documentId, { level: "info", op: "indexing.document.extracted" });
|
|
1290
|
+
logDocument(state, documentId, {
|
|
1291
|
+
level: "info",
|
|
1292
|
+
op: "indexing.document.skipped",
|
|
1293
|
+
extra: {
|
|
1294
|
+
reason: "unsupported",
|
|
1295
|
+
documentStatus: result.outcome.document.status,
|
|
1296
|
+
skippedDocuments: state.skippedDocuments,
|
|
1297
|
+
},
|
|
1298
|
+
});
|
|
1110
1299
|
yield {
|
|
1111
1300
|
kind: "document-extracted",
|
|
1112
1301
|
jobId: state.jobId,
|
|
@@ -1169,8 +1358,8 @@ async function* handlePersistedDocument(state, result) {
|
|
|
1169
1358
|
function* handleExtractionSkippedEvents(state, result) {
|
|
1170
1359
|
yield handleExtractionSkipped(state, result);
|
|
1171
1360
|
}
|
|
1172
|
-
function* handleExtractionFailedEvents(state, result) {
|
|
1173
|
-
yield handleExtractionFailed(state, result);
|
|
1361
|
+
function* handleExtractionFailedEvents(state, result, outcome) {
|
|
1362
|
+
yield handleExtractionFailed(state, result, outcome);
|
|
1174
1363
|
}
|
|
1175
1364
|
// Routes a file-extracted event: force-skipped docs are re-shaped to persisted so the
|
|
1176
1365
|
// standard chunk-and-embed pipeline runs on them.
|
|
@@ -1198,7 +1387,7 @@ async function* handleFileExtracted(state, result) {
|
|
|
1198
1387
|
return;
|
|
1199
1388
|
}
|
|
1200
1389
|
if (result.outcome.kind === "failed") {
|
|
1201
|
-
yield* handleExtractionFailedEvents(state, result);
|
|
1390
|
+
yield* handleExtractionFailedEvents(state, result, result.outcome);
|
|
1202
1391
|
return;
|
|
1203
1392
|
}
|
|
1204
1393
|
yield* handlePersistedDocument(state, result);
|
|
@@ -1206,7 +1395,13 @@ async function* handleFileExtracted(state, result) {
|
|
|
1206
1395
|
async function* handleDiscoveryEvent(state, source, evt) {
|
|
1207
1396
|
if (evt.kind === "file-discovered") {
|
|
1208
1397
|
state.totalDocuments += 1;
|
|
1209
|
-
|
|
1398
|
+
const documentId = documentIdFor({
|
|
1399
|
+
capsuleId: state.capsule.id,
|
|
1400
|
+
sourceId: source.id,
|
|
1401
|
+
relativePath: evt.relativePath,
|
|
1402
|
+
});
|
|
1403
|
+
captureRestoreSnapshotIfEligible(state, documentId);
|
|
1404
|
+
logDocumentExtractionStarted(state, documentId, evt.sizeBytes);
|
|
1210
1405
|
yield {
|
|
1211
1406
|
kind: "document-discovered",
|
|
1212
1407
|
jobId: state.jobId,
|
|
@@ -1218,24 +1413,7 @@ async function* handleDiscoveryEvent(state, source, evt) {
|
|
|
1218
1413
|
return;
|
|
1219
1414
|
}
|
|
1220
1415
|
if (evt.kind === "scope-error") {
|
|
1221
|
-
state
|
|
1222
|
-
state.discoveryFailedDocuments += 1;
|
|
1223
|
-
if (evt.error.code === "LIMIT_REACHED") {
|
|
1224
|
-
persistDiscoveryLimitWarning(state);
|
|
1225
|
-
}
|
|
1226
|
-
const err = {
|
|
1227
|
-
code: `DISCOVERY_FAILED:${evt.error.code}`,
|
|
1228
|
-
message: evt.error.message,
|
|
1229
|
-
};
|
|
1230
|
-
state.lastError = err;
|
|
1231
|
-
yield {
|
|
1232
|
-
kind: "document-failed",
|
|
1233
|
-
jobId: state.jobId,
|
|
1234
|
-
capsuleId: state.capsule.id,
|
|
1235
|
-
sourceId: source.id,
|
|
1236
|
-
...(evt.error.relativePath !== undefined ? { relativePath: evt.error.relativePath } : {}),
|
|
1237
|
-
error: err,
|
|
1238
|
-
};
|
|
1416
|
+
yield scopeErrorEvent(state, source, evt.error);
|
|
1239
1417
|
return;
|
|
1240
1418
|
}
|
|
1241
1419
|
if (evt.kind === "cancelled" || evt.kind === "completed") {
|
|
@@ -1249,6 +1427,70 @@ async function* handleDiscoveryEvent(state, source, evt) {
|
|
|
1249
1427
|
// reach that function — discard is a no-op if it already ran.
|
|
1250
1428
|
discardRestoreSnapshot(state, evt.result.outcome.document.id);
|
|
1251
1429
|
}
|
|
1430
|
+
// The discovery lane's progress tick AND the extraction-start marker, deliberately one line.
|
|
1431
|
+
// `discoverAndExtract` yields `file-discovered` and only THEN awaits that file's extraction, so
|
|
1432
|
+
// the instant this runs is both "the walk found another file" and "extraction of that file is
|
|
1433
|
+
// about to begin" — two lines here would carry the same timestamp and the same facts.
|
|
1434
|
+
//
|
|
1435
|
+
// It closes both halves of the field incident's silent middle. Between
|
|
1436
|
+
// `indexing.source.started` and `indexing.source.completed` a slow walk now advances a visible
|
|
1437
|
+
// counter instead of showing nothing, and a document that hangs mid-extraction leaves this line
|
|
1438
|
+
// with no `indexing.document.extracted` partner — which is what separates "the walk never
|
|
1439
|
+
// reached it" from "extraction started and never returned".
|
|
1440
|
+
//
|
|
1441
|
+
// `discoveredCount` is the running job-wide total: the "1" in "0 of 1 documents". The relative
|
|
1442
|
+
// path is a customer file name and is never written — the document digest carries identity, and
|
|
1443
|
+
// it is the same digest every later line for this document uses, so one grep follows the
|
|
1444
|
+
// document from here to its terminal line.
|
|
1445
|
+
function logDocumentExtractionStarted(state, documentId, sizeBytes) {
|
|
1446
|
+
logDocument(state, documentId, {
|
|
1447
|
+
level: "info",
|
|
1448
|
+
op: "indexing.document.extraction-started",
|
|
1449
|
+
extra: { discoveredCount: state.totalDocuments, sizeBytes },
|
|
1450
|
+
});
|
|
1451
|
+
}
|
|
1452
|
+
// Every walk-level rejection an operator can act on: a PATH_ESCAPE containment refusal, an
|
|
1453
|
+
// unreadable or unstattable directory, an invalid scope, and the LIMIT_REACHED truncation
|
|
1454
|
+
// frames. Each of them bumped two counters and produced a `document-failed` event that only a
|
|
1455
|
+
// consumer driving the iterator ever saw. `errorKind` is the discovery code; the message is
|
|
1456
|
+
// never written because a walk error's message quotes the path that produced it. Note that a
|
|
1457
|
+
// deny-list match is dropped inside the walk without an error at all and cannot reach here.
|
|
1458
|
+
function logDiscoveryScopeError(state, error) {
|
|
1459
|
+
logIndexing(state, {
|
|
1460
|
+
level: "warn",
|
|
1461
|
+
op: "indexing.discovery.scope-error",
|
|
1462
|
+
errorKind: error.code,
|
|
1463
|
+
extra: {
|
|
1464
|
+
scopedToFile: error.relativePath !== undefined,
|
|
1465
|
+
discoveryFailedDocuments: state.discoveryFailedDocuments,
|
|
1466
|
+
},
|
|
1467
|
+
});
|
|
1468
|
+
}
|
|
1469
|
+
// Returns the event rather than yielding it, so the caller keeps its `yield` direct. Delegating
|
|
1470
|
+
// with `yield*` from an async generator to a sync one routes every value through
|
|
1471
|
+
// AsyncFromSyncIterator and inserts extra microtask ticks between producing this event and the
|
|
1472
|
+
// cancellation re-check that follows it — a timing change this instrumentation must not make.
|
|
1473
|
+
function scopeErrorEvent(state, source, error) {
|
|
1474
|
+
state.failedDocuments += 1;
|
|
1475
|
+
state.discoveryFailedDocuments += 1;
|
|
1476
|
+
logDiscoveryScopeError(state, error);
|
|
1477
|
+
if (error.code === "LIMIT_REACHED") {
|
|
1478
|
+
persistDiscoveryLimitWarning(state);
|
|
1479
|
+
}
|
|
1480
|
+
const err = {
|
|
1481
|
+
code: `DISCOVERY_FAILED:${error.code}`,
|
|
1482
|
+
message: error.message,
|
|
1483
|
+
};
|
|
1484
|
+
state.lastError = err;
|
|
1485
|
+
return {
|
|
1486
|
+
kind: "document-failed",
|
|
1487
|
+
jobId: state.jobId,
|
|
1488
|
+
capsuleId: state.capsule.id,
|
|
1489
|
+
sourceId: source.id,
|
|
1490
|
+
...(error.relativePath !== undefined ? { relativePath: error.relativePath } : {}),
|
|
1491
|
+
error: err,
|
|
1492
|
+
};
|
|
1493
|
+
}
|
|
1252
1494
|
function shouldStopAfterEvent(event) {
|
|
1253
1495
|
return event.kind === "document-failed" && event.error.code === "INCOMPATIBLE_EMBEDDING_IDENTITY";
|
|
1254
1496
|
}
|
|
@@ -1263,8 +1505,8 @@ async function* streamDiscoveryEvent(state, source, evt) {
|
|
|
1263
1505
|
return false;
|
|
1264
1506
|
}
|
|
1265
1507
|
// ─── Per-source pipeline ──────────────────────────────────────────────────────
|
|
1266
|
-
|
|
1267
|
-
|
|
1508
|
+
function discoveryStreamFor(state, source) {
|
|
1509
|
+
return discoverAndExtract({
|
|
1268
1510
|
fs: state.options.workspaceFs,
|
|
1269
1511
|
store: state.options.store,
|
|
1270
1512
|
parserRegistry: state.options.parserRegistry,
|
|
@@ -1280,12 +1522,46 @@ async function* runOneSource(state, source) {
|
|
|
1280
1522
|
largeDocumentJobId: state.jobId,
|
|
1281
1523
|
chunkingStrategyVersion: chunkingStrategyKey(chunkingOptionsForState(state)),
|
|
1282
1524
|
}, sourceDiscoveryParams(state, source));
|
|
1525
|
+
}
|
|
1526
|
+
// Paired with `indexing.source.completed`. A walk that never returns — an unreachable network
|
|
1527
|
+
// mount, a directory tree that does not terminate — leaves this line with no partner, which is
|
|
1528
|
+
// the ONLY evidence that a run stalled in discovery rather than in embedding. The source id is
|
|
1529
|
+
// digested for the same reason the capsule id is: it is caller-supplied.
|
|
1530
|
+
function logSourceStarted(state, source, sourceDigest) {
|
|
1531
|
+
logIndexing(state, {
|
|
1532
|
+
level: "info",
|
|
1533
|
+
op: "indexing.source.started",
|
|
1534
|
+
extra: { sourceIdDigest: sourceDigest, scopeKind: source.scope.kind },
|
|
1535
|
+
});
|
|
1536
|
+
}
|
|
1537
|
+
// The discovered count is the number the field incident's operator was staring at ("0 of 1
|
|
1538
|
+
// documents"): it is what the walk actually found, as opposed to what got indexed.
|
|
1539
|
+
function logSourceCompleted(state, progress, counts) {
|
|
1540
|
+
logIndexing(state, {
|
|
1541
|
+
level: "info",
|
|
1542
|
+
op: "indexing.source.completed",
|
|
1543
|
+
durationMs: counts.durationMs,
|
|
1544
|
+
extra: {
|
|
1545
|
+
sourceIdDigest: counts.sourceDigest,
|
|
1546
|
+
discoveredCount: progress.discoveredPaths.size,
|
|
1547
|
+
failedCount: counts.failedCount,
|
|
1548
|
+
walkCompleted: progress.completed,
|
|
1549
|
+
cancelled: progress.cancelled,
|
|
1550
|
+
sawScopeError: progress.sawScopeError,
|
|
1551
|
+
},
|
|
1552
|
+
});
|
|
1553
|
+
}
|
|
1554
|
+
async function* runOneSource(state, source) {
|
|
1555
|
+
const stream = discoveryStreamFor(state, source);
|
|
1283
1556
|
const progress = {
|
|
1284
1557
|
cancelled: false,
|
|
1285
1558
|
sawScopeError: false,
|
|
1286
1559
|
completed: false,
|
|
1287
1560
|
discoveredPaths: new Set(),
|
|
1288
1561
|
};
|
|
1562
|
+
const sourceDigest = logDigest(String(source.id));
|
|
1563
|
+
const sourceElapsed = startKnowledgeLogTimer();
|
|
1564
|
+
logSourceStarted(state, source, sourceDigest);
|
|
1289
1565
|
// Snapshot the job-wide failure counter before this source's documents are processed so
|
|
1290
1566
|
// finalizeSourceRun can tell whether ANY document belonging to THIS source failed during this
|
|
1291
1567
|
// run, without conflating it with failures from other sources in the same multi-source job.
|
|
@@ -1307,7 +1583,13 @@ async function* runOneSource(state, source) {
|
|
|
1307
1583
|
break;
|
|
1308
1584
|
}
|
|
1309
1585
|
}
|
|
1310
|
-
|
|
1586
|
+
const failedCount = state.failedDocuments - failedDocumentsBefore;
|
|
1587
|
+
logSourceCompleted(state, progress, {
|
|
1588
|
+
sourceDigest,
|
|
1589
|
+
failedCount,
|
|
1590
|
+
durationMs: sourceElapsed(),
|
|
1591
|
+
});
|
|
1592
|
+
finalizeSourceRun(state, source, progress, failedCount);
|
|
1311
1593
|
}
|
|
1312
1594
|
function sourceDiscoveryParams(state, source) {
|
|
1313
1595
|
return {
|
|
@@ -1377,6 +1659,8 @@ function buildInitialState(options, capsule, sources, jobId, startedAt, tokenize
|
|
|
1377
1659
|
jobId,
|
|
1378
1660
|
capsule,
|
|
1379
1661
|
options,
|
|
1662
|
+
logContext: { jobId, capsuleIdDigest: logDigest(String(capsule.id)) },
|
|
1663
|
+
elapsed: startKnowledgeLogTimer(),
|
|
1380
1664
|
batchSize: clampBatchSize(options.batchSize),
|
|
1381
1665
|
concurrency: clampConcurrency(options.concurrency),
|
|
1382
1666
|
now: options.now ?? options.store._internal.now,
|
|
@@ -1480,37 +1764,101 @@ function cacheSuccessfulEmbeddingPreflight(cache, options, result, expiresAt) {
|
|
|
1480
1764
|
for (const key of keys)
|
|
1481
1765
|
cache.set(key, { result, expiresAt });
|
|
1482
1766
|
}
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
const
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1767
|
+
// Provider and model id are gateway configuration, not customer data, and they are the pair that
|
|
1768
|
+
// makes a preflight failure actionable. The endpoint is reduced to `scheme://host`.
|
|
1769
|
+
function preflightProbeExtra(adapter, options) {
|
|
1770
|
+
const host = embeddingEndpointHost(adapter.endpoint);
|
|
1771
|
+
return {
|
|
1772
|
+
provider: options.provider,
|
|
1773
|
+
modelId: options.modelId,
|
|
1774
|
+
...(options.expectedDimensions !== undefined
|
|
1775
|
+
? { expectedDimensions: options.expectedDimensions }
|
|
1776
|
+
: {}),
|
|
1777
|
+
fingerprinted: options.includeSpaceFingerprint === true,
|
|
1778
|
+
...(host === undefined ? {} : { endpointHost: host }),
|
|
1779
|
+
};
|
|
1780
|
+
}
|
|
1781
|
+
async function probeEmbeddingCapability(adapter, options, log) {
|
|
1782
|
+
const probe = preflightProbeExtra(adapter, options);
|
|
1783
|
+
log({ level: "info", op: "embedding.preflight.started", extra: { ...probe, cached: false } });
|
|
1784
|
+
const elapsed = startKnowledgeLogTimer();
|
|
1491
1785
|
const result = await verifyEmbeddingCapability(adapter, options);
|
|
1492
|
-
|
|
1786
|
+
const durationMs = elapsed();
|
|
1787
|
+
if (!result.ok) {
|
|
1788
|
+
log({
|
|
1789
|
+
level: "error",
|
|
1790
|
+
op: "embedding.preflight.failed",
|
|
1791
|
+
errorKind: result.reason,
|
|
1792
|
+
durationMs,
|
|
1793
|
+
extra: probe,
|
|
1794
|
+
});
|
|
1795
|
+
return result;
|
|
1796
|
+
}
|
|
1797
|
+
log({
|
|
1798
|
+
level: "info",
|
|
1799
|
+
op: "embedding.preflight.completed",
|
|
1800
|
+
durationMs,
|
|
1801
|
+
extra: { ...probe, observedDimensions: result.identity.vectorDimensions },
|
|
1802
|
+
});
|
|
1803
|
+
return result;
|
|
1804
|
+
}
|
|
1805
|
+
async function verifyEmbeddingPreflightCapability(adapter, options, cacheScope, now, log) {
|
|
1806
|
+
const cache = cacheScope === undefined ? undefined : embeddingPreflightCacheFor(cacheScope);
|
|
1807
|
+
const cached = cache?.get(embeddingPreflightCacheKey(options));
|
|
1808
|
+
if (cached !== undefined && cached.expiresAt > now()) {
|
|
1809
|
+
log({
|
|
1810
|
+
level: "info",
|
|
1811
|
+
op: "embedding.preflight.cache-hit",
|
|
1812
|
+
extra: { ...preflightProbeExtra(adapter, options), cached: true },
|
|
1813
|
+
});
|
|
1814
|
+
return cached.result;
|
|
1815
|
+
}
|
|
1816
|
+
const result = await probeEmbeddingCapability(adapter, options, log);
|
|
1817
|
+
if (cache === undefined || !result.ok)
|
|
1493
1818
|
return result;
|
|
1494
1819
|
cacheSuccessfulEmbeddingPreflight(cache, options, result, now() + EMBEDDING_PREFLIGHT_TTL_MS);
|
|
1495
1820
|
return result;
|
|
1496
1821
|
}
|
|
1822
|
+
// The preflight ANSWERED and the gateway's identity was still refused. This is the one outcome
|
|
1823
|
+
// an operator most often mistakes for an outage, so it gets its own line carrying both
|
|
1824
|
+
// dimensions — the whole diagnosis, and none of the safe message's prose.
|
|
1825
|
+
function logPreflightIdentityRejected(state, observed) {
|
|
1826
|
+
logEmbeddingRun(state, {
|
|
1827
|
+
level: "error",
|
|
1828
|
+
op: "embedding.preflight.identity-rejected",
|
|
1829
|
+
errorKind: "INCOMPATIBLE_EMBEDDING_IDENTITY",
|
|
1830
|
+
extra: {
|
|
1831
|
+
pinnedDimensions: state.capsule.embeddingModelIdentity.vectorDimensions,
|
|
1832
|
+
observedDimensions: observed.vectorDimensions,
|
|
1833
|
+
},
|
|
1834
|
+
});
|
|
1835
|
+
}
|
|
1836
|
+
function adoptPreflightIdentity(state, identity, op) {
|
|
1837
|
+
state.capsule = updateCapsuleEmbeddingModelIdentity(state.options.store, state.capsule.id, identity);
|
|
1838
|
+
logEmbeddingRun(state, {
|
|
1839
|
+
level: "info",
|
|
1840
|
+
op,
|
|
1841
|
+
extra: { observedDimensions: identity.vectorDimensions, provider: identity.provider },
|
|
1842
|
+
});
|
|
1843
|
+
}
|
|
1844
|
+
function embeddingPreflightSuccess(state, result) {
|
|
1845
|
+
if (hasProvisionalIdentity(state)) {
|
|
1846
|
+
adoptPreflightIdentity(state, result.identity, "embedding.preflight.identity-adopted");
|
|
1847
|
+
return undefined;
|
|
1848
|
+
}
|
|
1849
|
+
const compatibility = assertCompatibleEmbeddingIdentity(state.capsule.embeddingModelIdentity, result.identity);
|
|
1850
|
+
if (!compatibility.ok) {
|
|
1851
|
+
logPreflightIdentityRejected(state, result.identity);
|
|
1852
|
+
return { code: "INCOMPATIBLE_EMBEDDING_IDENTITY", message: compatibility.safeMessage };
|
|
1853
|
+
}
|
|
1854
|
+
if (embeddingIdentityChanged(state.capsule.embeddingModelIdentity, compatibility.identity)) {
|
|
1855
|
+
adoptPreflightIdentity(state, compatibility.identity, "embedding.preflight.identity-refreshed");
|
|
1856
|
+
}
|
|
1857
|
+
return undefined;
|
|
1858
|
+
}
|
|
1497
1859
|
function embeddingPreflightFailure(state, result) {
|
|
1498
1860
|
if (result.ok) {
|
|
1499
|
-
|
|
1500
|
-
state.capsule = updateCapsuleEmbeddingModelIdentity(state.options.store, state.capsule.id, result.identity);
|
|
1501
|
-
return undefined;
|
|
1502
|
-
}
|
|
1503
|
-
const compatibility = assertCompatibleEmbeddingIdentity(state.capsule.embeddingModelIdentity, result.identity);
|
|
1504
|
-
if (!compatibility.ok) {
|
|
1505
|
-
return {
|
|
1506
|
-
code: "INCOMPATIBLE_EMBEDDING_IDENTITY",
|
|
1507
|
-
message: compatibility.safeMessage,
|
|
1508
|
-
};
|
|
1509
|
-
}
|
|
1510
|
-
if (embeddingIdentityChanged(state.capsule.embeddingModelIdentity, compatibility.identity)) {
|
|
1511
|
-
state.capsule = updateCapsuleEmbeddingModelIdentity(state.options.store, state.capsule.id, compatibility.identity);
|
|
1512
|
-
}
|
|
1513
|
-
return undefined;
|
|
1861
|
+
return embeddingPreflightSuccess(state, result);
|
|
1514
1862
|
}
|
|
1515
1863
|
return {
|
|
1516
1864
|
code: result.reason === "dimension-mismatch"
|
|
@@ -1530,20 +1878,36 @@ function embeddingIdentityChanged(stored, current) {
|
|
|
1530
1878
|
stored.embeddingSpaceFingerprint !== current.embeddingSpaceFingerprint ||
|
|
1531
1879
|
stored.dimensionsParam !== current.dimensionsParam);
|
|
1532
1880
|
}
|
|
1881
|
+
// A THROWN preflight is the worst case for an operator: the adapter did not answer with a
|
|
1882
|
+
// classified failure, it blew up, and the run ends with a generic EMBEDDING_ADAPTER_FAILED whose
|
|
1883
|
+
// message says nothing about the cause. The kind — never the message, which carries the endpoint
|
|
1884
|
+
// and often the response body — is the whole difference between a DNS failure and a TLS refusal.
|
|
1885
|
+
function preflightThrowFailure(state, log, cause) {
|
|
1886
|
+
const cancelled = cancellationRequested(state) || (cause instanceof DOMException && cause.name === "AbortError");
|
|
1887
|
+
log({
|
|
1888
|
+
level: cancelled ? "warn" : "error",
|
|
1889
|
+
op: "embedding.preflight.failed",
|
|
1890
|
+
errorKind: cancelled ? "CANCELLED" : knowledgeErrorKind(cause),
|
|
1891
|
+
extra: { threw: true, ...endpointHostExtra(state) },
|
|
1892
|
+
});
|
|
1893
|
+
if (cancelled) {
|
|
1894
|
+
return { code: "CANCELLED", message: "indexing aborted via AbortSignal" };
|
|
1895
|
+
}
|
|
1896
|
+
return {
|
|
1897
|
+
code: "EMBEDDING_ADAPTER_FAILED",
|
|
1898
|
+
message: "embedding capability preflight failed before indexing started",
|
|
1899
|
+
};
|
|
1900
|
+
}
|
|
1533
1901
|
async function verifyEmbeddingPreflight(state) {
|
|
1902
|
+
const log = (event) => {
|
|
1903
|
+
logEmbeddingRun(state, event);
|
|
1904
|
+
};
|
|
1534
1905
|
try {
|
|
1535
|
-
const result = await verifyEmbeddingPreflightCapability(state.options.embeddingAdapter, embeddingPreflightOptions(state), state.options.embeddingPreflightCacheScope, state.now);
|
|
1906
|
+
const result = await verifyEmbeddingPreflightCapability(state.options.embeddingAdapter, embeddingPreflightOptions(state), state.options.embeddingPreflightCacheScope, state.now, log);
|
|
1536
1907
|
return embeddingPreflightFailure(state, result);
|
|
1537
1908
|
}
|
|
1538
1909
|
catch (cause) {
|
|
1539
|
-
|
|
1540
|
-
(cause instanceof DOMException && cause.name === "AbortError")) {
|
|
1541
|
-
return { code: "CANCELLED", message: "indexing aborted via AbortSignal" };
|
|
1542
|
-
}
|
|
1543
|
-
return {
|
|
1544
|
-
code: "EMBEDDING_ADAPTER_FAILED",
|
|
1545
|
-
message: "embedding capability preflight failed before indexing started",
|
|
1546
|
-
};
|
|
1910
|
+
return preflightThrowFailure(state, log, cause);
|
|
1547
1911
|
}
|
|
1548
1912
|
}
|
|
1549
1913
|
function modelUsePolicyPreflightFailure(capsule, options) {
|
|
@@ -1583,6 +1947,22 @@ function persistDiscoveryLimitWarning(state) {
|
|
|
1583
1947
|
if (state.discoveryLimitWarningPersisted)
|
|
1584
1948
|
return;
|
|
1585
1949
|
state.discoveryLimitWarningPersisted = true;
|
|
1950
|
+
// Truncation is the one discovery outcome where the run SUCCEEDS and the corpus is still
|
|
1951
|
+
// incomplete, so the operator has no failing signal to chase — they only notice later, when a
|
|
1952
|
+
// grounded answer is missing a document nobody knows was never indexed. The capsule diagnostic
|
|
1953
|
+
// below reaches the health surface; this line puts the same fact, plus the two caps that
|
|
1954
|
+
// produced it, in the file. Written once per run behind the guard above, because LIMIT_REACHED
|
|
1955
|
+
// surfaces once per ancestor frame.
|
|
1956
|
+
const discovery = resolvedDiscoveryOptions(state);
|
|
1957
|
+
logIndexing(state, {
|
|
1958
|
+
level: "warn",
|
|
1959
|
+
op: "indexing.discovery.limit-reached",
|
|
1960
|
+
extra: {
|
|
1961
|
+
discoveredCount: state.totalDocuments,
|
|
1962
|
+
maxFiles: discovery.maxFiles,
|
|
1963
|
+
maxDepth: discovery.maxDepth,
|
|
1964
|
+
},
|
|
1965
|
+
});
|
|
1586
1966
|
try {
|
|
1587
1967
|
insertDiagnosticRow(state.options.store._internal.db, {
|
|
1588
1968
|
id: state.idSource(),
|
|
@@ -1644,6 +2024,22 @@ function emitJobStarted(state, sources) {
|
|
|
1644
2024
|
jobId: state.jobId,
|
|
1645
2025
|
occurredAt: state.startedAt,
|
|
1646
2026
|
});
|
|
2027
|
+
// The first line of the run's spine, and the only place the run's shape is stated: how many
|
|
2028
|
+
// sources it will walk, and the two caps that decide the whole request profile. Everything
|
|
2029
|
+
// downstream is read against these numbers.
|
|
2030
|
+
logIndexing(state, {
|
|
2031
|
+
level: "info",
|
|
2032
|
+
op: "indexing.job.started",
|
|
2033
|
+
extra: {
|
|
2034
|
+
sourceCount: sources.length,
|
|
2035
|
+
batchSize: state.batchSize,
|
|
2036
|
+
concurrency: state.concurrency,
|
|
2037
|
+
force: state.options.force === true,
|
|
2038
|
+
resume: state.options.resume === true,
|
|
2039
|
+
contextualRetrieval: state.options.contextualRetrieval?.enabled === true,
|
|
2040
|
+
...endpointHostExtra(state),
|
|
2041
|
+
},
|
|
2042
|
+
});
|
|
1647
2043
|
return emit(state, event);
|
|
1648
2044
|
}
|
|
1649
2045
|
async function resolveIndexingTokenizer(options) {
|
|
@@ -1668,13 +2064,40 @@ async function* runSourcesWithProgress(state, sources) {
|
|
|
1668
2064
|
}
|
|
1669
2065
|
return identityFailure;
|
|
1670
2066
|
}
|
|
2067
|
+
// The run's prologue — capsule resolution, source resolution, the tokenizer load and the
|
|
2068
|
+
// started-job write — all happens BEFORE `indexing.job.started` can be emitted, and every one of
|
|
2069
|
+
// those steps can throw or hang: a missing capsule, a source filter matching nothing, a
|
|
2070
|
+
// tokenizer read off disk, a locked database. Until this line existed, a job that died or wedged
|
|
2071
|
+
// in its prologue produced exactly what a job that was never launched produces — nothing — and
|
|
2072
|
+
// no operator could tell the two apart. It is written before the first of those steps runs, and
|
|
2073
|
+
// it is the only line that cannot use `RunState`, because no state exists yet.
|
|
2074
|
+
//
|
|
2075
|
+
// The capsule id is caller-supplied, so it is digested here by the same function the rest of the
|
|
2076
|
+
// run uses: the prologue line and every later line share one correlation key.
|
|
2077
|
+
function logJobReceived(options, jobId) {
|
|
2078
|
+
emitKnowledgeLogEvent(options.logSink, {
|
|
2079
|
+
level: "info",
|
|
2080
|
+
category: "indexing",
|
|
2081
|
+
op: "indexing.job.received",
|
|
2082
|
+
correlationId: jobId,
|
|
2083
|
+
extra: {
|
|
2084
|
+
capsuleIdDigest: logDigest(String(options.capsuleId)),
|
|
2085
|
+
sourceIdFilterCount: options.sourceIds?.length ?? 0,
|
|
2086
|
+
force: options.force === true,
|
|
2087
|
+
resume: options.resume === true,
|
|
2088
|
+
},
|
|
2089
|
+
});
|
|
2090
|
+
}
|
|
1671
2091
|
// ─── Public entrypoint ────────────────────────────────────────────────────────
|
|
1672
2092
|
export async function* runIndexingJob(options) {
|
|
2093
|
+
// The job id is minted first so the prologue line can carry the correlation key every later
|
|
2094
|
+
// line uses. `idSource` is called exactly once here, as before.
|
|
2095
|
+
const idSource = options.idSource ?? (() => randomUUID());
|
|
2096
|
+
const jobId = idSource();
|
|
2097
|
+
logJobReceived(options, jobId);
|
|
1673
2098
|
const capsule = resolveCapsule(options);
|
|
1674
2099
|
const sources = resolveSources(options, capsule);
|
|
1675
2100
|
const startedAt = (options.now ?? options.store._internal.now)();
|
|
1676
|
-
const idSource = options.idSource ?? (() => randomUUID());
|
|
1677
|
-
const jobId = idSource();
|
|
1678
2101
|
const policyFailure = modelUsePolicyPreflightFailure(capsule, options);
|
|
1679
2102
|
const tokenizer = policyFailure === undefined
|
|
1680
2103
|
? await resolveIndexingTokenizer(options)
|
|
@@ -1806,6 +2229,36 @@ function resolveJobStatus(state, fatalFailure) {
|
|
|
1806
2229
|
}
|
|
1807
2230
|
return "succeeded";
|
|
1808
2231
|
}
|
|
2232
|
+
// The closing line of the run's spine, and the one an operator reads first. It restates every
|
|
2233
|
+
// counter the UI shows next to the terminal status and the total wall time, so "cancelled after
|
|
2234
|
+
// six minutes with zero vectors" is a single greppable line rather than an inference across the
|
|
2235
|
+
// whole file. `warn` for a cancellation and `error` for a failure: a run that did not succeed
|
|
2236
|
+
// must be visible at a level an operator filters TO, not one they filter out.
|
|
2237
|
+
const JOB_FINISH_LEVELS = {
|
|
2238
|
+
succeeded: "info",
|
|
2239
|
+
cancelled: "warn",
|
|
2240
|
+
failed: "error",
|
|
2241
|
+
};
|
|
2242
|
+
function logJobFinished(state, result) {
|
|
2243
|
+
logIndexing(state, {
|
|
2244
|
+
level: JOB_FINISH_LEVELS[result.status],
|
|
2245
|
+
op: "indexing.job.finished",
|
|
2246
|
+
durationMs: state.elapsed(),
|
|
2247
|
+
...(state.lastError !== undefined ? { errorKind: state.lastError.code } : {}),
|
|
2248
|
+
extra: {
|
|
2249
|
+
// NOT `status`. The sink flattens `extra` onto the same record as the envelope, whose
|
|
2250
|
+
// `status` field is the numeric HTTP status every operator query and dashboard filter
|
|
2251
|
+
// reads. A run's terminal state written under that name puts the string "succeeded" where
|
|
2252
|
+
// a number belongs and silently poisons those queries for every other line's `status`.
|
|
2253
|
+
jobStatus: result.status,
|
|
2254
|
+
totalDocuments: result.totalDocuments,
|
|
2255
|
+
processedDocuments: result.processedDocuments,
|
|
2256
|
+
failedDocuments: result.failedDocuments,
|
|
2257
|
+
skippedDocuments: result.skippedDocuments,
|
|
2258
|
+
vectorsPersisted: result.vectorsPersisted,
|
|
2259
|
+
},
|
|
2260
|
+
});
|
|
2261
|
+
}
|
|
1809
2262
|
function* finalize(state, fatalFailure) {
|
|
1810
2263
|
const finishedAt = state.now();
|
|
1811
2264
|
const status = resolveJobStatus(state, fatalFailure);
|
|
@@ -1823,6 +2276,7 @@ function* finalize(state, fatalFailure) {
|
|
|
1823
2276
|
// informational only — see the started block for the rationale
|
|
1824
2277
|
}
|
|
1825
2278
|
const result = buildResult(state, status, finishedAt);
|
|
2279
|
+
logJobFinished(state, result);
|
|
1826
2280
|
if (status === "cancelled") {
|
|
1827
2281
|
yield emit(state, { kind: "job-cancelled", jobId: state.jobId, result });
|
|
1828
2282
|
return;
|