@oscharko-dev/keiko 0.3.13 → 0.3.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ui/csp-hashes.json +6 -6
- package/dist/ui/static/404.html +1 -1
- package/dist/ui/static/__next.__PAGE__.txt +1 -1
- package/dist/ui/static/__next._full.txt +1 -1
- package/dist/ui/static/__next._head.txt +1 -1
- package/dist/ui/static/__next._index.txt +1 -1
- package/dist/ui/static/__next._tree.txt +1 -1
- package/dist/ui/static/_not-found/__next._full.txt +1 -1
- package/dist/ui/static/_not-found/__next._head.txt +1 -1
- package/dist/ui/static/_not-found/__next._index.txt +1 -1
- package/dist/ui/static/_not-found/__next._not-found.__PAGE__.txt +1 -1
- package/dist/ui/static/_not-found/__next._not-found.txt +1 -1
- package/dist/ui/static/_not-found/__next._tree.txt +1 -1
- package/dist/ui/static/_not-found.html +1 -1
- package/dist/ui/static/_not-found.txt +1 -1
- package/dist/ui/static/atlassian-connectors/__next._full.txt +1 -1
- package/dist/ui/static/atlassian-connectors/__next._head.txt +1 -1
- package/dist/ui/static/atlassian-connectors/__next._index.txt +1 -1
- package/dist/ui/static/atlassian-connectors/__next._tree.txt +1 -1
- package/dist/ui/static/atlassian-connectors/__next.atlassian-connectors.__PAGE__.txt +1 -1
- package/dist/ui/static/atlassian-connectors/__next.atlassian-connectors.txt +1 -1
- package/dist/ui/static/atlassian-connectors.html +1 -1
- package/dist/ui/static/atlassian-connectors.txt +1 -1
- package/dist/ui/static/index.html +1 -1
- package/dist/ui/static/index.txt +1 -1
- package/dist/ui/static/launch/__next._full.txt +1 -1
- package/dist/ui/static/launch/__next._head.txt +1 -1
- package/dist/ui/static/launch/__next._index.txt +1 -1
- package/dist/ui/static/launch/__next._tree.txt +1 -1
- package/dist/ui/static/launch/__next.launch.__PAGE__.txt +1 -1
- package/dist/ui/static/launch/__next.launch.txt +1 -1
- package/dist/ui/static/launch.html +1 -1
- package/dist/ui/static/launch.txt +1 -1
- package/dist/ui/static/local-knowledge/__next._full.txt +1 -1
- package/dist/ui/static/local-knowledge/__next._head.txt +1 -1
- package/dist/ui/static/local-knowledge/__next._index.txt +1 -1
- package/dist/ui/static/local-knowledge/__next._tree.txt +1 -1
- package/dist/ui/static/local-knowledge/__next.local-knowledge.__PAGE__.txt +1 -1
- package/dist/ui/static/local-knowledge/__next.local-knowledge.txt +1 -1
- package/dist/ui/static/local-knowledge/capsule/__next._full.txt +1 -1
- package/dist/ui/static/local-knowledge/capsule/__next._head.txt +1 -1
- package/dist/ui/static/local-knowledge/capsule/__next._index.txt +1 -1
- package/dist/ui/static/local-knowledge/capsule/__next._tree.txt +1 -1
- package/dist/ui/static/local-knowledge/capsule/__next.local-knowledge.capsule.__PAGE__.txt +1 -1
- package/dist/ui/static/local-knowledge/capsule/__next.local-knowledge.capsule.txt +1 -1
- package/dist/ui/static/local-knowledge/capsule/__next.local-knowledge.txt +1 -1
- package/dist/ui/static/local-knowledge/capsule.html +1 -1
- package/dist/ui/static/local-knowledge/capsule.txt +1 -1
- package/dist/ui/static/local-knowledge.html +1 -1
- package/dist/ui/static/local-knowledge.txt +1 -1
- package/node_modules/@oscharko-dev/keiko-cli/dist/ui.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-cli/dist/ui.js +25 -6
- package/node_modules/@oscharko-dev/keiko-cli/package.json +13 -13
- package/node_modules/@oscharko-dev/keiko-connectors/package.json +3 -3
- package/node_modules/@oscharko-dev/keiko-contracts/dist/index.d.ts +2 -2
- package/node_modules/@oscharko-dev/keiko-contracts/dist/index.js +2 -2
- package/node_modules/@oscharko-dev/keiko-contracts/package.json +1 -1
- package/node_modules/@oscharko-dev/keiko-evaluations/package.json +9 -9
- package/node_modules/@oscharko-dev/keiko-evidence/dist/version.d.ts +1 -1
- package/node_modules/@oscharko-dev/keiko-evidence/dist/version.js +1 -1
- package/node_modules/@oscharko-dev/keiko-evidence/package.json +4 -4
- package/node_modules/@oscharko-dev/keiko-git/package.json +1 -1
- package/node_modules/@oscharko-dev/keiko-harness/dist/version.d.ts +1 -1
- package/node_modules/@oscharko-dev/keiko-harness/dist/version.js +1 -1
- package/node_modules/@oscharko-dev/keiko-harness/package.json +6 -6
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/connector-pod.d.ts +2 -0
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/connector-pod.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/connector-pod.js +1 -0
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/index.d.ts +1 -0
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/index.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/index.js +3 -0
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/indexing/embedding-batcher.d.ts +1 -0
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/indexing/embedding-batcher.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/indexing/embedding-batcher.js +341 -62
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/indexing/orchestrator.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/indexing/orchestrator.js +586 -132
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/indexing/types.d.ts +9 -0
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/indexing/types.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/knowledge-log.d.ts +20 -0
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/knowledge-log.d.ts.map +1 -0
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/knowledge-log.js +154 -0
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/manual-pod-refresh.d.ts +2 -0
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/manual-pod-refresh.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/manual-pod-refresh.js +1 -0
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/manual-pod.d.ts +2 -0
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/manual-pod.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/manual-pod.js +1 -0
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/repository-pod.d.ts +2 -0
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/repository-pod.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/repository-pod.js +1 -0
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/store.d.ts +2 -0
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/store.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/store.js +43 -1
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/version.d.ts +1 -1
- package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/version.js +1 -1
- package/node_modules/@oscharko-dev/keiko-local-knowledge/package.json +5 -5
- package/node_modules/@oscharko-dev/keiko-memory-capture/dist/version.d.ts +1 -1
- package/node_modules/@oscharko-dev/keiko-memory-capture/dist/version.js +1 -1
- package/node_modules/@oscharko-dev/keiko-memory-capture/package.json +3 -3
- package/node_modules/@oscharko-dev/keiko-memory-consolidation/dist/version.d.ts +1 -1
- package/node_modules/@oscharko-dev/keiko-memory-consolidation/dist/version.js +1 -1
- package/node_modules/@oscharko-dev/keiko-memory-consolidation/package.json +3 -3
- package/node_modules/@oscharko-dev/keiko-memory-governance/dist/version.d.ts +1 -1
- package/node_modules/@oscharko-dev/keiko-memory-governance/dist/version.js +1 -1
- package/node_modules/@oscharko-dev/keiko-memory-governance/package.json +3 -3
- package/node_modules/@oscharko-dev/keiko-memory-retrieval/dist/version.d.ts +1 -1
- package/node_modules/@oscharko-dev/keiko-memory-retrieval/dist/version.js +1 -1
- package/node_modules/@oscharko-dev/keiko-memory-retrieval/package.json +3 -3
- package/node_modules/@oscharko-dev/keiko-memory-vault/dist/version.d.ts +1 -1
- package/node_modules/@oscharko-dev/keiko-memory-vault/dist/version.js +1 -1
- package/node_modules/@oscharko-dev/keiko-memory-vault/package.json +3 -3
- package/node_modules/@oscharko-dev/keiko-model-gateway/dist/gateway.d.ts +15 -2
- package/node_modules/@oscharko-dev/keiko-model-gateway/dist/gateway.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-model-gateway/dist/gateway.js +147 -15
- package/node_modules/@oscharko-dev/keiko-model-gateway/dist/http.d.ts +3 -0
- package/node_modules/@oscharko-dev/keiko-model-gateway/dist/http.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-model-gateway/dist/http.js +188 -23
- package/node_modules/@oscharko-dev/keiko-model-gateway/dist/observability.d.ts +27 -0
- package/node_modules/@oscharko-dev/keiko-model-gateway/dist/observability.d.ts.map +1 -0
- package/node_modules/@oscharko-dev/keiko-model-gateway/dist/observability.js +180 -0
- package/node_modules/@oscharko-dev/keiko-model-gateway/dist/openai-embedding-adapter.d.ts +5 -0
- package/node_modules/@oscharko-dev/keiko-model-gateway/dist/openai-embedding-adapter.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-model-gateway/dist/openai-embedding-adapter.js +339 -55
- package/node_modules/@oscharko-dev/keiko-model-gateway/dist/resilience.d.ts +18 -5
- package/node_modules/@oscharko-dev/keiko-model-gateway/dist/resilience.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-model-gateway/dist/resilience.js +141 -26
- package/node_modules/@oscharko-dev/keiko-model-gateway/dist/version.d.ts +1 -1
- package/node_modules/@oscharko-dev/keiko-model-gateway/dist/version.js +1 -1
- package/node_modules/@oscharko-dev/keiko-model-gateway/package.json +3 -3
- package/node_modules/@oscharko-dev/keiko-quality-intelligence/package.json +3 -3
- package/node_modules/@oscharko-dev/keiko-sandbox/package.json +2 -2
- package/node_modules/@oscharko-dev/keiko-sdk/package.json +9 -9
- package/node_modules/@oscharko-dev/keiko-security/dist/version.d.ts +1 -1
- package/node_modules/@oscharko-dev/keiko-security/dist/version.js +1 -1
- package/node_modules/@oscharko-dev/keiko-security/package.json +2 -2
- package/node_modules/@oscharko-dev/keiko-server/dist/atlassian/syncService.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-server/dist/atlassian/syncService.js +4 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/bounded-request-body.d.ts +1 -1
- package/node_modules/@oscharko-dev/keiko-server/dist/bounded-request-body.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-server/dist/bounded-request-body.js +51 -3
- package/node_modules/@oscharko-dev/keiko-server/dist/diagnostics-log.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-server/dist/diagnostics-log.js +77 -3
- package/node_modules/@oscharko-dev/keiko-server/dist/gateway-instance-cache.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-server/dist/gateway-instance-cache.js +71 -11
- package/node_modules/@oscharko-dev/keiko-server/dist/gateway-setup.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-server/dist/gateway-setup.js +9 -1
- package/node_modules/@oscharko-dev/keiko-server/dist/grounded-qa.js +1 -1
- package/node_modules/@oscharko-dev/keiko-server/dist/grounded-rerank-facade.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-server/dist/grounded-rerank-facade.js +37 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/index.d.ts +1 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/index.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-server/dist/index.js +7 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/local-knowledge-handlers.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-server/dist/local-knowledge-handlers.js +158 -36
- package/node_modules/@oscharko-dev/keiko-server/dist/local-knowledge-remediation-store.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-server/dist/local-knowledge-remediation-store.js +9 -1
- package/node_modules/@oscharko-dev/keiko-server/dist/local-knowledge-remediation.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-server/dist/local-knowledge-remediation.js +10 -5
- package/node_modules/@oscharko-dev/keiko-server/dist/local-knowledge-store-open.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-server/dist/local-knowledge-store-open.js +8 -2
- package/node_modules/@oscharko-dev/keiko-server/dist/manual/manual-pod-service.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-server/dist/manual/manual-pod-service.js +3 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/memory-embedding.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-server/dist/memory-embedding.js +135 -36
- package/node_modules/@oscharko-dev/keiko-server/dist/observability/index.d.ts +3 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/observability/index.d.ts.map +1 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/observability/index.js +9 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/observability/log-level.d.ts +13 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/observability/log-level.d.ts.map +1 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/observability/log-level.js +57 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/observability/log-redaction.d.ts +20 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/observability/log-redaction.d.ts.map +1 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/observability/log-redaction.js +499 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/observability/route-template.d.ts +4 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/observability/route-template.d.ts.map +1 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/observability/route-template.js +340 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/observability/server-log.d.ts +49 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/observability/server-log.d.ts.map +1 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/observability/server-log.js +449 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/observability/server-logger.d.ts +36 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/observability/server-logger.d.ts.map +1 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/observability/server-logger.js +190 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/process-log-sink.d.ts +11 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/process-log-sink.d.ts.map +1 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/process-log-sink.js +48 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/server.d.ts +2 -0
- package/node_modules/@oscharko-dev/keiko-server/dist/server.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-server/dist/server.js +22 -4
- package/node_modules/@oscharko-dev/keiko-server/dist/store-handlers.d.ts.map +1 -1
- package/node_modules/@oscharko-dev/keiko-server/dist/store-handlers.js +7 -2
- package/node_modules/@oscharko-dev/keiko-server/package.json +21 -21
- package/node_modules/@oscharko-dev/keiko-tools/dist/version.d.ts +1 -1
- package/node_modules/@oscharko-dev/keiko-tools/dist/version.js +1 -1
- package/node_modules/@oscharko-dev/keiko-tools/package.json +5 -5
- package/node_modules/@oscharko-dev/keiko-verification/package.json +5 -5
- package/node_modules/@oscharko-dev/keiko-workflows/package.json +10 -10
- package/node_modules/@oscharko-dev/keiko-workspace/dist/version.d.ts +1 -1
- package/node_modules/@oscharko-dev/keiko-workspace/dist/version.js +1 -1
- package/node_modules/@oscharko-dev/keiko-workspace/package.json +3 -3
- package/package.json +25 -25
- package/release-impact.catalog.json +117 -7
- package/vendor/{oscharko-dev-keiko-cli-0.3.13.tgz → oscharko-dev-keiko-cli-0.3.15.tgz} +0 -0
- package/vendor/{oscharko-dev-keiko-connectors-0.3.13.tgz → oscharko-dev-keiko-connectors-0.3.15.tgz} +0 -0
- package/vendor/{oscharko-dev-keiko-contracts-0.3.13.tgz → oscharko-dev-keiko-contracts-0.3.15.tgz} +0 -0
- package/vendor/{oscharko-dev-keiko-evaluations-0.3.13.tgz → oscharko-dev-keiko-evaluations-0.3.15.tgz} +0 -0
- package/vendor/{oscharko-dev-keiko-evidence-0.3.13.tgz → oscharko-dev-keiko-evidence-0.3.15.tgz} +0 -0
- package/vendor/{oscharko-dev-keiko-git-0.3.13.tgz → oscharko-dev-keiko-git-0.3.15.tgz} +0 -0
- package/vendor/{oscharko-dev-keiko-harness-0.3.13.tgz → oscharko-dev-keiko-harness-0.3.15.tgz} +0 -0
- package/vendor/oscharko-dev-keiko-local-knowledge-0.3.15.tgz +0 -0
- package/vendor/{oscharko-dev-keiko-memory-capture-0.3.13.tgz → oscharko-dev-keiko-memory-capture-0.3.15.tgz} +0 -0
- package/vendor/{oscharko-dev-keiko-memory-consolidation-0.3.13.tgz → oscharko-dev-keiko-memory-consolidation-0.3.15.tgz} +0 -0
- package/vendor/oscharko-dev-keiko-memory-governance-0.3.15.tgz +0 -0
- package/vendor/{oscharko-dev-keiko-memory-retrieval-0.3.13.tgz → oscharko-dev-keiko-memory-retrieval-0.3.15.tgz} +0 -0
- package/vendor/{oscharko-dev-keiko-memory-vault-0.3.13.tgz → oscharko-dev-keiko-memory-vault-0.3.15.tgz} +0 -0
- package/vendor/oscharko-dev-keiko-model-gateway-0.3.15.tgz +0 -0
- package/vendor/{oscharko-dev-keiko-quality-intelligence-0.3.13.tgz → oscharko-dev-keiko-quality-intelligence-0.3.15.tgz} +0 -0
- package/vendor/{oscharko-dev-keiko-sandbox-0.3.13.tgz → oscharko-dev-keiko-sandbox-0.3.15.tgz} +0 -0
- package/vendor/oscharko-dev-keiko-sdk-0.3.15.tgz +0 -0
- package/vendor/{oscharko-dev-keiko-security-0.3.13.tgz → oscharko-dev-keiko-security-0.3.15.tgz} +0 -0
- package/vendor/oscharko-dev-keiko-server-0.3.15.tgz +0 -0
- package/vendor/{oscharko-dev-keiko-tools-0.3.13.tgz → oscharko-dev-keiko-tools-0.3.15.tgz} +0 -0
- package/vendor/{oscharko-dev-keiko-verification-0.3.13.tgz → oscharko-dev-keiko-verification-0.3.15.tgz} +0 -0
- package/vendor/{oscharko-dev-keiko-workflows-0.3.13.tgz → oscharko-dev-keiko-workflows-0.3.15.tgz} +0 -0
- package/vendor/{oscharko-dev-keiko-workspace-0.3.13.tgz → oscharko-dev-keiko-workspace-0.3.15.tgz} +0 -0
- package/vendor/oscharko-dev-keiko-local-knowledge-0.3.13.tgz +0 -0
- package/vendor/oscharko-dev-keiko-memory-governance-0.3.13.tgz +0 -0
- package/vendor/oscharko-dev-keiko-model-gateway-0.3.13.tgz +0 -0
- package/vendor/oscharko-dev-keiko-sdk-0.3.13.tgz +0 -0
- package/vendor/oscharko-dev-keiko-server-0.3.13.tgz +0 -0
- /package/dist/ui/static/_next/static/{iwhpaHGb663ka5R3lhZBj → 4j3JDaPBERRu_ZIgeR1VQ}/_buildManifest.js +0 -0
- /package/dist/ui/static/_next/static/{iwhpaHGb663ka5R3lhZBj → 4j3JDaPBERRu_ZIgeR1VQ}/_clientMiddlewareManifest.js +0 -0
- /package/dist/ui/static/_next/static/{iwhpaHGb663ka5R3lhZBj → 4j3JDaPBERRu_ZIgeR1VQ}/_ssgManifest.js +0 -0
|
@@ -16,7 +16,88 @@ import { assertCompatibleEmbeddingIdentity, EMBEDDING_NORMALIZATION, l2Normalize
|
|
|
16
16
|
import { composeVectorRecord, insertVectorRow, invalidateVectorIndexStateForCapsules, } from "./vector-persist.js";
|
|
17
17
|
import { IndexingError, } from "./types.js";
|
|
18
18
|
import { chunkDedupeKey } from "../chunking/chunker.js";
|
|
19
|
+
import { emitKnowledgeLogEvent, knowledgeErrorKind, startKnowledgeLogTimer, } from "../knowledge-log.js";
|
|
19
20
|
import { conservativeTokenEstimatorTokenizer } from "../chunking/token-estimator.js";
|
|
21
|
+
// ─── Activity log ────────────────────────────────────────────────────────────
|
|
22
|
+
// Every line this module writes is content-free: counts, byte-free sizes, durations, HTTP
|
|
23
|
+
// statuses, error kinds, and vector dimensions. Chunk text and embeddings never appear. The
|
|
24
|
+
// sink is optional and defaults to nothing, so an unwired caller pays a single undefined check.
|
|
25
|
+
// `scheme://host[:port]` and nothing else — no userinfo, no path, no query, no fragment. The
|
|
26
|
+
// endpoint is what separates "the operator pointed the Pod at a gateway that is not there" from
|
|
27
|
+
// "the configured gateway is slow", and it is the field an operator reads first next to
|
|
28
|
+
// `durationMs`; the path and query are where a provider URL historically carried a deployment
|
|
29
|
+
// id, an api-version, or an outright credential, so they are dropped rather than redacted. An
|
|
30
|
+
// unparseable endpoint yields `undefined` instead of echoing the raw string back.
|
|
31
|
+
//
|
|
32
|
+
// This deliberately mirrors `logEndpointHost` in `keiko-model-gateway/src/observability.ts`,
|
|
33
|
+
// which is not on that package's public barrel — importing it would mean widening another
|
|
34
|
+
// package's exported surface (and its packaged-surface contract) from here.
|
|
35
|
+
export function embeddingEndpointHost(endpoint) {
|
|
36
|
+
if (endpoint === undefined)
|
|
37
|
+
return undefined;
|
|
38
|
+
try {
|
|
39
|
+
const parsed = new URL(endpoint);
|
|
40
|
+
return `${parsed.protocol}//${parsed.host}`;
|
|
41
|
+
}
|
|
42
|
+
catch {
|
|
43
|
+
return undefined;
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
// The correlation fields ride in `extra` rather than replacing `correlationId`, which carries
|
|
47
|
+
// the job id alone so an operator can grep one run out of a file interleaving four of them.
|
|
48
|
+
function correlationExtra(context) {
|
|
49
|
+
if (context === undefined)
|
|
50
|
+
return {};
|
|
51
|
+
return {
|
|
52
|
+
capsuleIdDigest: context.capsuleIdDigest,
|
|
53
|
+
...(context.documentIdDigest !== undefined
|
|
54
|
+
? { documentIdDigest: context.documentIdDigest }
|
|
55
|
+
: {}),
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
function logEmbedding(options, event) {
|
|
59
|
+
const sink = options.logSink;
|
|
60
|
+
if (sink === undefined)
|
|
61
|
+
return;
|
|
62
|
+
const context = options.logContext;
|
|
63
|
+
const extra = { ...correlationExtra(context), ...event.extra };
|
|
64
|
+
// A throwing sink must not escape into the retry ladder, the pinned-identity gate, or
|
|
65
|
+
// `persistAndReport`: those read their own control flow from what the adapter returned, and a
|
|
66
|
+
// logging failure landing there would be accounted as an embedding failure. `emitKnowledgeLogEvent`
|
|
67
|
+
// also makes a permanently failing sink report itself once — see `knowledge-log.ts`.
|
|
68
|
+
emitKnowledgeLogEvent(sink, {
|
|
69
|
+
...event,
|
|
70
|
+
category: "embedding",
|
|
71
|
+
...(context !== undefined ? { correlationId: context.jobId } : {}),
|
|
72
|
+
...(Object.keys(extra).length > 0 ? { extra } : {}),
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
// Omitted rather than written as `undefined` when the endpoint does not parse: a field that is
|
|
76
|
+
// present but empty reads as "the gateway has no host", which is a different diagnosis.
|
|
77
|
+
function endpointExtra(options) {
|
|
78
|
+
const host = embeddingEndpointHost(options.adapter.endpoint);
|
|
79
|
+
return host === undefined ? {} : { endpointHost: host };
|
|
80
|
+
}
|
|
81
|
+
// One shape for both transports' retry lines. The `ok` guard is what narrows the union down to
|
|
82
|
+
// the failure variants that actually carry `kind` and `status`.
|
|
83
|
+
//
|
|
84
|
+
// `durationMs` and `endpointHost` are on every one of them because without the pair a hang is
|
|
85
|
+
// unreadable: "refused instantly" (a gateway that is not listening, ~0 ms) and "burned the full
|
|
86
|
+
// provider deadline" (a gateway that accepted the connection and never answered, ~60 s) produce
|
|
87
|
+
// the SAME error kind, and telling them apart is the whole diagnosis during the six-minute stall
|
|
88
|
+
// this instrumentation exists for.
|
|
89
|
+
function logEmbeddingRetry(options, outcome, op, durationMs, extra) {
|
|
90
|
+
if (outcome.ok)
|
|
91
|
+
return;
|
|
92
|
+
logEmbedding(options, {
|
|
93
|
+
level: "warn",
|
|
94
|
+
op,
|
|
95
|
+
errorKind: outcome.kind,
|
|
96
|
+
status: outcome.status,
|
|
97
|
+
durationMs,
|
|
98
|
+
extra: { ...extra, ...endpointExtra(options) },
|
|
99
|
+
});
|
|
100
|
+
}
|
|
20
101
|
// ─── Concurrency primitive ───────────────────────────────────────────────────
|
|
21
102
|
// Hand-rolled bounded-concurrency runner. Avoids pulling in `p-limit` (the local-knowledge
|
|
22
103
|
// package's runtime-deps surface stays narrow per ADR-0019-3e). Order of `inputs` is
|
|
@@ -46,7 +127,7 @@ async function runBounded(inputs, concurrency, work) {
|
|
|
46
127
|
// adapter's `signal`, `apiKeyHeaderName`, and timeout defaults are honoured via the
|
|
47
128
|
// optional-spread pattern (the strict `exactOptionalPropertyTypes` mode forbids passing
|
|
48
129
|
// `undefined` for an optional property).
|
|
49
|
-
async function embedSingleChunkWithModel(adapter, chunk, pinnedIdentity, signal) {
|
|
130
|
+
async function embedSingleChunkWithModel(adapter, chunk, pinnedIdentity, signal, logContext) {
|
|
50
131
|
return adapter.request({
|
|
51
132
|
endpoint: adapter.endpoint,
|
|
52
133
|
apiKey: adapter.apiKey,
|
|
@@ -59,6 +140,8 @@ async function embedSingleChunkWithModel(adapter, chunk, pinnedIdentity, signal)
|
|
|
59
140
|
? { dimensions: pinnedIdentity.dimensionsParam }
|
|
60
141
|
: {}),
|
|
61
142
|
...(signal !== undefined ? { signal } : {}),
|
|
143
|
+
// Same join key as the batch path above.
|
|
144
|
+
...(logContext === undefined ? {} : { logContext: { correlationId: logContext.jobId } }),
|
|
62
145
|
});
|
|
63
146
|
}
|
|
64
147
|
// ─── Transient-failure retry ─────────────────────────────────────────────────
|
|
@@ -111,22 +194,47 @@ function resolveRetry(retry) {
|
|
|
111
194
|
sleep: retry?.sleep ?? defaultSleep,
|
|
112
195
|
};
|
|
113
196
|
}
|
|
197
|
+
async function timedAttempt(call) {
|
|
198
|
+
const elapsed = startKnowledgeLogTimer();
|
|
199
|
+
const outcome = await call();
|
|
200
|
+
return { outcome, durationMs: elapsed() };
|
|
201
|
+
}
|
|
202
|
+
// `attempt` MEANS THE SAME THING ON BOTH TRANSPORTS: adapter round-trips issued so far. It is a
|
|
203
|
+
// field an operator greps across the whole log, so it may not be a loop index here and a count
|
|
204
|
+
// there. This path used to log the loop's retry number, and on exhaustion `retry.maxRetries` — the
|
|
205
|
+
// BUDGET, which is the exact defect the array-batch counter in `BatchRetryState` was introduced to
|
|
206
|
+
// fix, and it read one lower than the round-trips actually spent because the first call is not a
|
|
207
|
+
// retry. `maxRetries` remains the separate budget field, so "2 of 3 round-trips, budget 2" stays
|
|
208
|
+
// readable on one line.
|
|
114
209
|
async function embedChunkWithRetry(options, chunk) {
|
|
115
210
|
const retry = resolveRetry(options.retry);
|
|
116
|
-
let
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
211
|
+
let attempts = 0;
|
|
212
|
+
const attemptOnce = async () => {
|
|
213
|
+
attempts += 1;
|
|
214
|
+
return timedAttempt(async () => embedSingleChunkWithModel(options.adapter, chunk, options.pinnedIdentity, options.signal, options.logContext));
|
|
215
|
+
};
|
|
216
|
+
let attempted = await attemptOnce();
|
|
217
|
+
for (let retryNumber = 1; retryNumber <= retry.maxRetries; retryNumber += 1) {
|
|
218
|
+
if (!isTransientOutcome(attempted.outcome) || options.signal?.aborted === true) {
|
|
219
|
+
return attempted.outcome;
|
|
120
220
|
}
|
|
221
|
+
const delayMs = backoffMs(retryNumber, retry.baseDelayMs);
|
|
222
|
+
logEmbeddingRetry(options, attempted.outcome, "embedding.chunk.retry", attempted.durationMs, {
|
|
223
|
+
attempt: attempts,
|
|
224
|
+
maxRetries: retry.maxRetries,
|
|
225
|
+
delayMs,
|
|
226
|
+
transport: "scalar",
|
|
227
|
+
});
|
|
121
228
|
try {
|
|
122
|
-
await retry.sleep(
|
|
229
|
+
await retry.sleep(delayMs, options.signal);
|
|
123
230
|
}
|
|
124
231
|
catch {
|
|
125
|
-
return outcome; // aborted mid-backoff; the abort gate converts this to CANCELLED
|
|
232
|
+
return attempted.outcome; // aborted mid-backoff; the abort gate converts this to CANCELLED
|
|
126
233
|
}
|
|
127
|
-
|
|
234
|
+
attempted = await attemptOnce();
|
|
128
235
|
}
|
|
129
|
-
|
|
236
|
+
logEmbeddingRetry(options, attempted.outcome, "embedding.chunk.retry-exhausted", attempted.durationMs, { attempt: attempts, maxRetries: retry.maxRetries, transport: "scalar" });
|
|
237
|
+
return attempted.outcome;
|
|
130
238
|
}
|
|
131
239
|
// ─── Array-batch embedding (#189 GRD-004) ────────────────────────────────────
|
|
132
240
|
// When the adapter exposes `requestBatch`, embed many unique chunks per HTTP round-trip.
|
|
@@ -202,6 +310,13 @@ function batchRequestFor(options, inputs) {
|
|
|
202
310
|
? { dimensions: options.pinnedIdentity.dimensionsParam }
|
|
203
311
|
: {}),
|
|
204
312
|
...(options.signal !== undefined ? { signal: options.signal } : {}),
|
|
313
|
+
// The gateway writes ~150 of the ~168 lines an indexing incident produces. Without this the
|
|
314
|
+
// outbound half of the file carries no correlation id and cannot be joined to the knowledge
|
|
315
|
+
// lines that caused it — two disjoint islands the moment concurrency puts a second document
|
|
316
|
+
// in flight. The job id is a uuid and is safe verbatim.
|
|
317
|
+
...(options.logContext === undefined
|
|
318
|
+
? {}
|
|
319
|
+
: { logContext: { correlationId: options.logContext.jobId } }),
|
|
205
320
|
};
|
|
206
321
|
}
|
|
207
322
|
// A transient failure that carries a completed prefix is PROGRESS: absorb it and retry only the
|
|
@@ -223,41 +338,94 @@ function mergeCompletedPrefix(completed, outcome) {
|
|
|
223
338
|
return outcome;
|
|
224
339
|
return { ok: true, value: [...completed, ...outcome.value] };
|
|
225
340
|
}
|
|
341
|
+
// Absorb any completed prefix, spend or reset the retry budget, record the decision, and
|
|
342
|
+
// return the backoff to wait before the next attempt. A retry that made progress is a
|
|
343
|
+
// DIFFERENT event from one that did not: only the second kind can exhaust the budget, and an
|
|
344
|
+
// operator watching a long batch needs to tell "advancing slowly" from "wedged".
|
|
345
|
+
function advanceBatchRetry(options, attempted, state, retry) {
|
|
346
|
+
const advanced = absorbPartialProgress(attempted.outcome, state.completed, state.remaining);
|
|
347
|
+
const progressed = advanced !== undefined;
|
|
348
|
+
if (advanced === undefined) {
|
|
349
|
+
state.zeroProgressRetries += 1;
|
|
350
|
+
}
|
|
351
|
+
else {
|
|
352
|
+
state.remaining = advanced;
|
|
353
|
+
state.zeroProgressRetries = 0;
|
|
354
|
+
}
|
|
355
|
+
const delayMs = backoffMs(progressed ? 1 : state.zeroProgressRetries, retry.baseDelayMs);
|
|
356
|
+
logEmbeddingRetry(options, attempted.outcome, progressed ? "embedding.batch.partial-progress" : "embedding.batch.retry", attempted.durationMs, {
|
|
357
|
+
attempt: state.attempts,
|
|
358
|
+
zeroProgressRetries: state.zeroProgressRetries,
|
|
359
|
+
maxRetries: retry.maxRetries,
|
|
360
|
+
delayMs,
|
|
361
|
+
remainingCount: state.remaining.length,
|
|
362
|
+
completedCount: state.completed.length,
|
|
363
|
+
transport: "array-batch",
|
|
364
|
+
});
|
|
365
|
+
return delayMs;
|
|
366
|
+
}
|
|
226
367
|
async function embedArrayBatchWithRetry(options, inputs) {
|
|
227
368
|
const requestBatch = options.adapter.requestBatch;
|
|
228
369
|
if (requestBatch === undefined) {
|
|
370
|
+
logEmbedding(options, {
|
|
371
|
+
level: "debug",
|
|
372
|
+
op: "embedding.batch.transport-unavailable",
|
|
373
|
+
extra: { itemCount: inputs.length, transport: "array-batch" },
|
|
374
|
+
});
|
|
229
375
|
return { ok: false, kind: "transport" };
|
|
230
376
|
}
|
|
231
377
|
const retry = resolveRetry(options.retry);
|
|
232
|
-
const
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
378
|
+
const state = {
|
|
379
|
+
completed: [],
|
|
380
|
+
remaining: inputs,
|
|
381
|
+
zeroProgressRetries: 0,
|
|
382
|
+
attempts: 0,
|
|
383
|
+
};
|
|
384
|
+
const attemptOnce = async () => {
|
|
385
|
+
state.attempts += 1;
|
|
386
|
+
return timedAttempt(async () => requestBatch(batchRequestFor(options, state.remaining)));
|
|
387
|
+
};
|
|
388
|
+
let attempted = await attemptOnce();
|
|
389
|
+
while (state.zeroProgressRetries < retry.maxRetries) {
|
|
390
|
+
if (!isTransientBatchOutcome(attempted.outcome) || options.signal?.aborted === true) {
|
|
238
391
|
break;
|
|
239
392
|
}
|
|
240
|
-
const
|
|
241
|
-
if (advanced === undefined) {
|
|
242
|
-
zeroProgressRetries += 1;
|
|
243
|
-
}
|
|
244
|
-
else {
|
|
245
|
-
remaining = advanced;
|
|
246
|
-
zeroProgressRetries = 0;
|
|
247
|
-
}
|
|
393
|
+
const delayMs = advanceBatchRetry(options, attempted, state, retry);
|
|
248
394
|
try {
|
|
249
|
-
await retry.sleep(
|
|
395
|
+
await retry.sleep(delayMs, options.signal);
|
|
250
396
|
}
|
|
251
397
|
catch {
|
|
252
398
|
break; // aborted mid-backoff; the abort gate converts this to CANCELLED
|
|
253
399
|
}
|
|
254
|
-
|
|
400
|
+
attempted = await attemptOnce();
|
|
255
401
|
}
|
|
256
|
-
return mergeCompletedPrefix(completed, outcome);
|
|
402
|
+
return mergeCompletedPrefix(state.completed, attempted.outcome);
|
|
257
403
|
}
|
|
258
404
|
// Apply the per-vector identity gate exactly as the scalar path does. Order-independent:
|
|
259
405
|
// once `state.identityFailure` is set, embedChunkBatch persists nothing, so which concurrent
|
|
260
406
|
// batch first observes the drift is irrelevant to the outcome.
|
|
407
|
+
// Both transports reach the identity gate, and both must record the rejection identically —
|
|
408
|
+
// so the state mutation AND its log line live here rather than being duplicated at each call
|
|
409
|
+
// site. The safe message is deliberately NOT logged: the line carries the two dimensions that
|
|
410
|
+
// explain the drift, which is the whole diagnosis and none of the prose.
|
|
411
|
+
function recordIdentityFailure(options, state, observed, safeMessage) {
|
|
412
|
+
const failure = {
|
|
413
|
+
code: "INCOMPATIBLE_EMBEDDING_IDENTITY",
|
|
414
|
+
message: safeMessage,
|
|
415
|
+
};
|
|
416
|
+
state.identityFailure = failure;
|
|
417
|
+
logEmbedding(options, {
|
|
418
|
+
level: "error",
|
|
419
|
+
op: "embedding.identity.rejected",
|
|
420
|
+
errorKind: failure.code,
|
|
421
|
+
extra: {
|
|
422
|
+
pinnedDimensions: options.pinnedIdentity.vectorDimensions,
|
|
423
|
+
observedDimensions: observed.vectorDimensions,
|
|
424
|
+
pinnedNormalization: options.pinnedIdentity.normalization ?? EMBEDDING_NORMALIZATION,
|
|
425
|
+
},
|
|
426
|
+
});
|
|
427
|
+
return failure;
|
|
428
|
+
}
|
|
261
429
|
function gateVectorOutcome(representative, success, options, state) {
|
|
262
430
|
if (state.identityFailure !== undefined) {
|
|
263
431
|
return { ok: false, chunk: representative, error: state.identityFailure };
|
|
@@ -265,11 +433,8 @@ function gateVectorOutcome(representative, success, options, state) {
|
|
|
265
433
|
const observed = identityFromAdapter(options.pinnedIdentity, success);
|
|
266
434
|
const compat = assertCompatibleEmbeddingIdentity(options.pinnedIdentity, observed);
|
|
267
435
|
if (!compat.ok) {
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
message: compat.safeMessage,
|
|
271
|
-
};
|
|
272
|
-
return { ok: false, chunk: representative, error: state.identityFailure };
|
|
436
|
+
const failure = recordIdentityFailure(options, state, observed, compat.safeMessage);
|
|
437
|
+
return { ok: false, chunk: representative, error: failure };
|
|
273
438
|
}
|
|
274
439
|
return { ok: true, chunk: representative, success };
|
|
275
440
|
}
|
|
@@ -282,9 +447,26 @@ async function embedUniqueBatch(batch, options, state) {
|
|
|
282
447
|
if (abortError !== undefined) {
|
|
283
448
|
return batch.map((r) => ({ ok: false, chunk: r.representative, error: abortError }));
|
|
284
449
|
}
|
|
285
|
-
const
|
|
450
|
+
const attempted = await timedAttempt(async () => embedArrayBatchWithRetry(options, batch.map((r) => r.representative.text)));
|
|
451
|
+
const outcome = attempted.outcome;
|
|
286
452
|
if (!outcome.ok) {
|
|
287
453
|
const error = errorFromKind(outcome.kind, outcome.status);
|
|
454
|
+
// The duration here spans the whole retry ladder for this batch — backoff included — which
|
|
455
|
+
// is deliberately a different number from the per-attempt duration on the retry lines: it
|
|
456
|
+
// is how long the RUN spent on these items before giving up on them.
|
|
457
|
+
logEmbedding(options, {
|
|
458
|
+
level: "warn",
|
|
459
|
+
op: "embedding.batch.failed",
|
|
460
|
+
errorKind: outcome.kind,
|
|
461
|
+
status: outcome.status,
|
|
462
|
+
durationMs: attempted.durationMs,
|
|
463
|
+
extra: {
|
|
464
|
+
itemCount: batch.length,
|
|
465
|
+
transient: isTransientFailure(outcome.kind, outcome.status),
|
|
466
|
+
transport: "array-batch",
|
|
467
|
+
...endpointExtra(options),
|
|
468
|
+
},
|
|
469
|
+
});
|
|
288
470
|
return batch.map((r) => ({ ok: false, chunk: r.representative, error }));
|
|
289
471
|
}
|
|
290
472
|
return batch.map((request, i) => {
|
|
@@ -380,11 +562,8 @@ async function buildUniqueChunkOutcome(request, options, state) {
|
|
|
380
562
|
const observed = identityFromAdapter(options.pinnedIdentity, outcome.value);
|
|
381
563
|
const compat = assertCompatibleEmbeddingIdentity(options.pinnedIdentity, observed);
|
|
382
564
|
if (!compat.ok) {
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
message: compat.safeMessage,
|
|
386
|
-
};
|
|
387
|
-
return { ok: false, chunk: request.representative, error: state.identityFailure };
|
|
565
|
+
const failure = recordIdentityFailure(options, state, observed, compat.safeMessage);
|
|
566
|
+
return { ok: false, chunk: request.representative, error: failure };
|
|
388
567
|
}
|
|
389
568
|
return { ok: true, chunk: request.representative, success: outcome.value };
|
|
390
569
|
}
|
|
@@ -401,32 +580,69 @@ function expandUniqueOutcomes(uniqueRequests, uniqueOutcomes) {
|
|
|
401
580
|
}
|
|
402
581
|
return outcomes;
|
|
403
582
|
}
|
|
583
|
+
// Array-batch path: collapse the unique requests into ceil(N / itemCap) HTTP calls, run those
|
|
584
|
+
// calls with bounded concurrency, then flatten back into request order. A tokenizer failure
|
|
585
|
+
// while budgeting degrades the WHOLE set to errors without a single request being issued —
|
|
586
|
+
// a silent outcome from the event stream's point of view, so it gets its own line.
|
|
587
|
+
async function embedViaArrayBatches(uniqueRequests, options, state) {
|
|
588
|
+
let batches;
|
|
589
|
+
try {
|
|
590
|
+
batches = groupIntoBatches(uniqueRequests, options);
|
|
591
|
+
}
|
|
592
|
+
catch (cause) {
|
|
593
|
+
logEmbedding(options, {
|
|
594
|
+
level: "warn",
|
|
595
|
+
op: "embedding.batch.budgeting-failed",
|
|
596
|
+
errorKind: knowledgeErrorKind(cause),
|
|
597
|
+
extra: { uniqueChunkCount: uniqueRequests.length },
|
|
598
|
+
});
|
|
599
|
+
return tokenBudgetErrorOutcomes(uniqueRequests, cause);
|
|
600
|
+
}
|
|
601
|
+
// Info, not debug: the number of HTTP round-trips this flush will issue is the run's request
|
|
602
|
+
// profile. One batch of 96 that never answers and 96 batches of one that each answer are the
|
|
603
|
+
// same chunk count and completely different investigations, and an operator reading the file
|
|
604
|
+
// at the default level must be able to tell which one is in front of them.
|
|
605
|
+
logEmbedding(options, {
|
|
606
|
+
level: "info",
|
|
607
|
+
op: "embedding.batch.grouped",
|
|
608
|
+
extra: {
|
|
609
|
+
uniqueChunkCount: uniqueRequests.length,
|
|
610
|
+
batchCount: batches.length,
|
|
611
|
+
concurrency: options.concurrency,
|
|
612
|
+
...endpointExtra(options),
|
|
613
|
+
},
|
|
614
|
+
});
|
|
615
|
+
const batchOutcomes = await runBounded(batches, options.concurrency, async (batch) => embedUniqueBatch(batch, options, state));
|
|
616
|
+
return batchOutcomes.flat();
|
|
617
|
+
}
|
|
404
618
|
// Build all per-chunk outcomes BEFORE we open a write transaction. The identity gate runs
|
|
405
619
|
// after every successful response so we fail fast on dimension mismatch.
|
|
406
620
|
async function buildChunkOutcomes(chunks, options) {
|
|
407
621
|
const state = { identityFailure: undefined };
|
|
408
622
|
const uniqueRequests = dedupeEmbeddingRequests(chunks);
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
623
|
+
const arrayBatchCapable = typeof options.adapter.requestBatch === "function";
|
|
624
|
+
// Info, not debug. The transport choice is invisible in the job's event stream, yet it decides
|
|
625
|
+
// the whole request profile of the run: scalar issues one HTTP call per unique chunk where
|
|
626
|
+
// array-batch issues one per ~96, so "36 vectors, no progress" means something different under
|
|
627
|
+
// each. The dedupe ratio beside it explains a request count that does not match the chunk
|
|
628
|
+
// count, which otherwise reads as lost work. One line per flush, not per chunk — the per-chunk
|
|
629
|
+
// detail stays at debug.
|
|
630
|
+
logEmbedding(options, {
|
|
631
|
+
level: "info",
|
|
632
|
+
op: "embedding.batch.transport-selected",
|
|
633
|
+
extra: {
|
|
634
|
+
transport: arrayBatchCapable ? "array-batch" : "scalar",
|
|
635
|
+
chunkCount: chunks.length,
|
|
636
|
+
uniqueChunkCount: uniqueRequests.length,
|
|
637
|
+
dedupedCount: chunks.length - uniqueRequests.length,
|
|
638
|
+
concurrency: options.concurrency,
|
|
639
|
+
...endpointExtra(options),
|
|
640
|
+
},
|
|
641
|
+
});
|
|
642
|
+
// Scalar fallback (adapters/stubs without `requestBatch`): one HTTP call per unique chunk.
|
|
643
|
+
const uniqueOutcomes = arrayBatchCapable
|
|
644
|
+
? await embedViaArrayBatches(uniqueRequests, options, state)
|
|
645
|
+
: await runBounded(uniqueRequests, options.concurrency, async (request) => buildUniqueChunkOutcome(request, options, state));
|
|
430
646
|
const outcomes = expandUniqueOutcomes(uniqueRequests, uniqueOutcomes);
|
|
431
647
|
return state.identityFailure === undefined
|
|
432
648
|
? { outcomes }
|
|
@@ -472,23 +688,86 @@ function persistOutcomes(store, outcomes, pinnedIdentity, idSource, now) {
|
|
|
472
688
|
}
|
|
473
689
|
return persisted;
|
|
474
690
|
}
|
|
475
|
-
//
|
|
691
|
+
// One shape for every way a batch can end, so an operator can grep a single op prefix and see
|
|
692
|
+
// how many chunks went in, how many vectors came out, and how long it took.
|
|
693
|
+
function logBatchClosed(options, level, op, counts) {
|
|
694
|
+
logEmbedding(options, {
|
|
695
|
+
level,
|
|
696
|
+
op,
|
|
697
|
+
errorKind: counts.errorKind,
|
|
698
|
+
durationMs: counts.durationMs,
|
|
699
|
+
extra: {
|
|
700
|
+
chunkCount: counts.chunkCount,
|
|
701
|
+
vectorCount: counts.vectorCount,
|
|
702
|
+
errorCount: counts.errorCount,
|
|
703
|
+
},
|
|
704
|
+
});
|
|
705
|
+
}
|
|
706
|
+
// Vector persistence is the LAST step of a flush and the only one that throws instead of
|
|
707
|
+
// returning an error: `persistOutcomes` opens a transaction, and a failing insert (a disk-full
|
|
708
|
+
// sqlite write, a foreign-key violation from a concurrently deleted chunk row) propagates out of
|
|
709
|
+
// `embedChunkBatch` past every closing line below. The run then ends with an exception the
|
|
710
|
+
// orchestrator turns into a document failure, and the activity log shows a batch that started,
|
|
711
|
+
// grouped, embedded — and simply stopped, with the successful gateway round-trip still recorded
|
|
712
|
+
// as if it were the last thing that happened. This is the one line that names the write.
|
|
713
|
+
function persistOutcomesLogged(options, outcomes, chunkCount, elapsed) {
|
|
714
|
+
try {
|
|
715
|
+
return persistOutcomes(options.store, outcomes, options.pinnedIdentity, options.idSource, options.now);
|
|
716
|
+
}
|
|
717
|
+
catch (cause) {
|
|
718
|
+
logBatchClosed(options, "error", "embedding.batch.persist-failed", {
|
|
719
|
+
chunkCount,
|
|
720
|
+
vectorCount: 0,
|
|
721
|
+
errorCount: outcomes.filter((outcome) => !outcome.ok).length,
|
|
722
|
+
durationMs: elapsed(),
|
|
723
|
+
errorKind: knowledgeErrorKind(cause),
|
|
724
|
+
});
|
|
725
|
+
throw cause;
|
|
726
|
+
}
|
|
727
|
+
}
|
|
728
|
+
function persistAndReport(options, outcomes, errors, chunkCount, elapsed) {
|
|
729
|
+
const vectors = persistOutcomesLogged(options, outcomes, chunkCount, elapsed);
|
|
730
|
+
const durationMs = elapsed();
|
|
731
|
+
logBatchClosed(options, errors.length === 0 ? "info" : "warn", "embedding.batch.completed", {
|
|
732
|
+
chunkCount,
|
|
733
|
+
vectorCount: vectors.length,
|
|
734
|
+
errorCount: errors.length,
|
|
735
|
+
durationMs,
|
|
736
|
+
});
|
|
737
|
+
return { vectors, errors };
|
|
738
|
+
}
|
|
476
739
|
export async function embedChunkBatch(chunks, options) {
|
|
477
740
|
if (chunks.length === 0) {
|
|
478
741
|
return { vectors: [], errors: [] };
|
|
479
742
|
}
|
|
743
|
+
const elapsed = startKnowledgeLogTimer();
|
|
480
744
|
const { outcomes, identityFailure } = await buildChunkOutcomes(chunks, options);
|
|
481
745
|
const errors = outcomes
|
|
482
746
|
.filter((o) => !o.ok)
|
|
483
747
|
.map((o) => o.error);
|
|
484
|
-
// Identity drift OR cancellation: refuse to persist ANY row from this batch.
|
|
748
|
+
// Identity drift OR cancellation: refuse to persist ANY row from this batch. Both are
|
|
749
|
+
// fail-closed rejections that discard completed work, so both are logged at the level that
|
|
750
|
+
// says so rather than being inferred from a missing "completed" line.
|
|
485
751
|
if (identityFailure !== undefined) {
|
|
752
|
+
logBatchClosed(options, "error", "embedding.batch.rejected", {
|
|
753
|
+
chunkCount: chunks.length,
|
|
754
|
+
vectorCount: 0,
|
|
755
|
+
errorCount: errors.length,
|
|
756
|
+
durationMs: elapsed(),
|
|
757
|
+
errorKind: identityFailure.code,
|
|
758
|
+
});
|
|
486
759
|
return { vectors: [], errors };
|
|
487
760
|
}
|
|
488
761
|
const abortError = checkAbort(options.signal);
|
|
489
762
|
if (abortError !== undefined) {
|
|
763
|
+
logBatchClosed(options, "warn", "embedding.batch.cancelled", {
|
|
764
|
+
chunkCount: chunks.length,
|
|
765
|
+
vectorCount: 0,
|
|
766
|
+
errorCount: errors.length + 1,
|
|
767
|
+
durationMs: elapsed(),
|
|
768
|
+
errorKind: abortError.code,
|
|
769
|
+
});
|
|
490
770
|
return { vectors: [], errors: [...errors, abortError] };
|
|
491
771
|
}
|
|
492
|
-
|
|
493
|
-
return { vectors, errors };
|
|
772
|
+
return persistAndReport(options, outcomes, errors, chunks.length, elapsed);
|
|
494
773
|
}
|
package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/indexing/orchestrator.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"orchestrator.d.ts","sourceRoot":"","sources":["../../src/indexing/orchestrator.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"orchestrator.d.ts","sourceRoot":"","sources":["../../src/indexing/orchestrator.ts"],"names":[],"mappings":"AAsIA,OAAO,EAML,KAAK,aAAa,EAElB,KAAK,eAAe,EAErB,MAAM,YAAY,CAAC;AAqpFpB,eAAO,MAAM,4BAA4B,4BAA4B,CAAC;AAoKtE,wBAAuB,cAAc,CAAC,OAAO,EAAE,eAAe,GAAG,aAAa,CAAC,aAAa,CAAC,CAwC5F;AAQD,eAAO,MAAM,mCAAmC,IAAI,CAAC;AAKrD,eAAO,MAAM,kCAAkC,kCACoD,CAAC"}
|