@fgv/ts-agent-memory 5.1.0-50 → 5.1.0-52
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/packlets/retrieve/fragmentSemanticRetriever.js +94 -5
- package/dist/packlets/retrieve/fragmentSemanticRetriever.js.map +1 -1
- package/dist/packlets/store/fileTreeMemoryStore.js +36 -123
- package/dist/packlets/store/fileTreeMemoryStore.js.map +1 -1
- package/dist/packlets/store/memoryStore.js.map +1 -1
- package/dist/packlets/store/storeFileAccess.js +99 -0
- package/dist/packlets/store/storeFileAccess.js.map +1 -0
- package/dist/packlets/store/storeIdentity.js +88 -0
- package/dist/packlets/store/storeIdentity.js.map +1 -0
- package/dist/packlets/types/identityResolver.js +6 -0
- package/dist/packlets/types/identityResolver.js.map +1 -0
- package/dist/packlets/types/index.js +1 -0
- package/dist/packlets/types/index.js.map +1 -1
- package/dist/packlets/vector/inMemoryFragmentCosineIndex.js +38 -17
- package/dist/packlets/vector/inMemoryFragmentCosineIndex.js.map +1 -1
- package/dist/packlets/vector/vectorIndex.js.map +1 -1
- package/dist/ts-agent-memory.d.ts +188 -41
- package/lib/packlets/retrieve/fragmentSemanticRetriever.d.ts +79 -1
- package/lib/packlets/retrieve/fragmentSemanticRetriever.d.ts.map +1 -1
- package/lib/packlets/retrieve/fragmentSemanticRetriever.js +94 -5
- package/lib/packlets/retrieve/fragmentSemanticRetriever.js.map +1 -1
- package/lib/packlets/store/fileTreeMemoryStore.d.ts +4 -19
- package/lib/packlets/store/fileTreeMemoryStore.d.ts.map +1 -1
- package/lib/packlets/store/fileTreeMemoryStore.js +36 -123
- package/lib/packlets/store/fileTreeMemoryStore.js.map +1 -1
- package/lib/packlets/store/memoryStore.d.ts +2 -2
- package/lib/packlets/store/memoryStore.d.ts.map +1 -1
- package/lib/packlets/store/memoryStore.js.map +1 -1
- package/lib/packlets/store/storeFileAccess.d.ts +40 -0
- package/lib/packlets/store/storeFileAccess.d.ts.map +1 -0
- package/lib/packlets/store/storeFileAccess.js +105 -0
- package/lib/packlets/store/storeFileAccess.js.map +1 -0
- package/lib/packlets/store/storeIdentity.d.ts +63 -0
- package/lib/packlets/store/storeIdentity.d.ts.map +1 -0
- package/lib/packlets/store/storeIdentity.js +94 -0
- package/lib/packlets/store/storeIdentity.js.map +1 -0
- package/lib/packlets/types/identityResolver.d.ts +42 -0
- package/lib/packlets/types/identityResolver.d.ts.map +1 -0
- package/lib/packlets/types/identityResolver.js +7 -0
- package/lib/packlets/types/identityResolver.js.map +1 -0
- package/lib/packlets/types/index.d.ts +1 -0
- package/lib/packlets/types/index.d.ts.map +1 -1
- package/lib/packlets/types/index.js +1 -0
- package/lib/packlets/types/index.js.map +1 -1
- package/lib/packlets/vector/inMemoryFragmentCosineIndex.d.ts +9 -17
- package/lib/packlets/vector/inMemoryFragmentCosineIndex.d.ts.map +1 -1
- package/lib/packlets/vector/inMemoryFragmentCosineIndex.js +38 -17
- package/lib/packlets/vector/inMemoryFragmentCosineIndex.js.map +1 -1
- package/lib/packlets/vector/vectorIndex.d.ts +57 -6
- package/lib/packlets/vector/vectorIndex.d.ts.map +1 -1
- package/lib/packlets/vector/vectorIndex.js.map +1 -1
- package/package.json +7 -7
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"memoryStore.js","sourceRoot":"","sources":["../../../src/packlets/store/memoryStore.ts"],"names":[],"mappings":"AAAA;;;GAGG","sourcesContent":["/*\n * Copyright (c) 2026 Erik Fortune\n * SPDX-License-Identifier: MIT\n */\n\nimport { Result } from '@fgv/ts-utils';\nimport {\n DedupScope,\n EntityId,\n IMemoryRecord,\n IMemoryRecordResolver,\n Kind,\n MemoryId,\n MemoryScopeKey\n} from '../types';\nimport { IIndexedMemoryEntry } from '../index';\nimport { IMemoryRecordSource, IScopedMemoryRecord } from '../vector';\nimport { MemoryListSelection } from './listSelection';\nimport { IDerivedStateCoverage } from './coverage';\nimport { DerivedArtifact, ReconcileReport } from './reconcile';\n\n/**\n * The writable, FileTree-backed, content-hash-deduped memory store.\n *\n * @remarks\n * Extends {@link IMemoryRecordResolver} because the retrievers take a resolver\n * and a store is the obvious one to hand them — `{ index, resolver: store }` is\n * the documented wiring, and it has to type-check for the `IMemoryStore` handle\n * a consumer actually holds, not only for the concrete `FileTreeMemoryStore`.\n * @public\n */\nexport interface IMemoryStore extends IMemoryRecordResolver {\n /**\n * Keyed read by entity id. Resolves `entityId` to a storage address via the\n * registered {@link IIdentityCodec} for `kind`. Returns `undefined` when no\n * record exists. For a versioned (temporal) kind the current version is\n * *selected* from the derived in-memory index — the version history is walked\n * over envelopes, never bodies — and then that one version is read from\n * storage. So the selection costs no file reads and the result costs exactly\n * one, rather than one per version.\n */\n get(kind: Kind, entityId: EntityId): Promise<Result<IMemoryRecord<unknown> | undefined>>;\n\n /**\n * Direct read by `(scope, MemoryId)`. Returns `undefined` when not found.\n */\n getById(scope: MemoryScopeKey, id: MemoryId): Promise<Result<IMemoryRecord<unknown> | undefined>>;\n\n /**\n * List records: select over the derived index, then materialize the survivors.\n *\n * @remarks\n * **The selection is required and must narrow.** Since the index holds\n * envelopes only, every returned record is read from storage — so a call that\n * narrows nothing reads the whole vault, and that has to be a decision rather\n * than a default. Omitting the argument is a compile error; passing one with no\n * `scope` / `kind` / `tag` fails with a message naming {@link scanEveryRecord},\n * which is how a caller says it meant it.\n *\n * The requirement buys **explicitness, not a cost bound** — `{ kind }` on a\n * vault dominated by that kind still materializes most of it. What it prevents\n * is the whole-vault read nobody chose.\n *\n * If you only need to select, use {@link IMemoryStore.listEntries} instead: no\n * selection, no file reads.\n */\n list(selection: MemoryListSelection): Promise<Result<ReadonlyArray<IMemoryRecord<unknown>>>>;\n\n /**\n * How much of the store's **derived state** exists, resolved by {@link Kind}.\n *\n * @remarks\n * Answers *\"is my derived state consistent with my records, and if not by how\n * much?\"* for every artifact the store derives — `rank`, record vectors, and\n * fragment vectors — in one call.\n *\n * **Cheap and total, by contract rather than by implementation.** Every input is\n * an envelope field or an index-side count: it reads **no record bodies** and\n * calls **no embedder**, and the walk over the vault's own state touches the\n * filesystem not at all. That is why it takes no selection, unlike\n * {@link IMemoryStore.list} — the guard there exists because an unnarrowed list\n * reads the vault, and putting one here would decorate a free operation and make\n * that guard mean less.\n *\n * **The index-side counts are the one exception, and it is the caller's own\n * index that spends it.** A persistent index answers `size` / `recordCount` /\n * `fragmentCount` with a query — `SqliteVecVectorIndex` runs a prepared `COUNT`\n * — so on a durable backend this call does I/O and can **fail**. It is bounded\n * (one count per wired index, never per record) and it is why the return is a\n * `Result` rather than a bare value.\n *\n * *If a future addition to the report would require reading a record body, it\n * does not belong on this report.*\n *\n * The counterpart is {@link IMemoryStore.reconcile}: coverage says **how big**\n * the gap is, cheaply; reconcile says **what it was** and closes what it can, at\n * the cost of re-running the embedder. Neither substitutes for the other.\n *\n * **A lane is reported here whenever its *index* is wired**, which is weaker\n * than what reconcile requires (index **and** embedder). That is intended: an\n * index without an embedder still holds vectors and still answers queries, so\n * its coverage is a real number worth reporting. The consequence to expect is a\n * half-wired store that reports a gap `reconcile` will refuse to close, naming\n * the missing embedder.\n */\n coverage(): Promise<Result<IDerivedStateCoverage>>;\n\n /**\n * Every entry in the vault — scope and envelope, **no bodies**. Reads no files\n * and requires no selection, because there is nothing to be careful about: it\n * returns what the index already holds.\n *\n * @remarks\n * This is the whole-vault read most callers actually want. Selection, grouping,\n * counting, \"which kinds are in here\", \"what links at this\" — all of it is\n * envelope work. Reach for {@link IMemoryStore.list} with\n * {@link scanEveryRecord} only when you genuinely need every body.\n *\n * Synchronous in spirit but `Promise`-returning for consistency with the rest\n * of the store surface.\n */\n listEntries(): Promise<Result<ReadonlyArray<IIndexedMemoryEntry>>>;\n\n /**\n * List EVERY record in the vault, each paired with its scope-qualified\n * `(scope, id)` address — the projection {@link IMemoryRecordSource} requires.\n * Unlike {@link IMemoryStore.list | list}, it takes no filter (whole-vault) and\n * returns {@link IScopedMemoryRecord}s so a re-index keys each entry on the same\n * scoped target the incremental embed-on-write path uses. Two records that share\n * a filename stem across scopes appear as distinct entries.\n */\n listScoped(): Promise<Result<ReadonlyArray<IScopedMemoryRecord>>>;\n\n /**\n * Adapt this store to the {@link IMemoryRecordSource} seam so it can drive\n * {@link IVectorIndex} rebuilds (e.g. `InMemoryCosineIndex.rebuild`). The\n * returned source's `list()` delegates to {@link IMemoryStore.listScoped},\n * **filtered to the kinds {@link IMemoryStore.embedsKind | embedsKind} reports** —\n * this source exists to feed the record vector index, so a kind excluded from\n * that index is excluded here too, and a reopen does not re-embed records the\n * index will never return. With no\n * {@link IFileTreeMemoryStoreCreateParams.embedKinds | embedKinds} declaration\n * every kind participates and the filter is the identity. `listScoped` itself is\n * **not** filtered and remains the whole-vault surface.\n *\n * The filter also **counts what it drops**, onto\n * {@link IMemoryRecordListing.excluded} — this is the layer where the exclusion\n * decision is made and so the only one that can. The count is always present\n * (empty when nothing was excluded).\n *\n * The store cannot implement {@link IMemoryRecordSource} directly because its\n * `list(filter?)` returns bare records (the ergonomic query surface) while the\n * seam's `list()` returns scope-qualified records.\n */\n asRecordSource(): IMemoryRecordSource;\n\n /**\n * The EFFECTIVE {@link DedupScope} for `kind` — the granularity at which a\n * write for this kind deduplicates against the existing vault.\n *\n * @remarks\n * This is a **read accessor over the store's already-injected write policies**,\n * and it is the single place any caller — the store's own write path included —\n * asks what a kind's dedup granularity is. It resolves the full chain the store\n * applies on write: the kind's registered {@link IWritePolicy}, falling back to\n * the store's default policy, then that policy's\n * {@link IWritePolicy.dedupScope | dedupScope}, falling back to\n * {@link DEFAULT_DEDUP_SCOPE}. Note the store's default policy is a\n * {@link KnowledgeLwwPolicy}, which declares `'content'` — so a kind with NO\n * registered policy resolves to `'content'`, not to `DEFAULT_DEDUP_SCOPE`.\n *\n * It exists so a caller that must agree with the store about dedup granularity\n * — notably the ingest orchestrator's stage-4 layer-1 exact match — can read the\n * declaration through this seam instead of being handed a second copy of the\n * policy map. A second declaration site is precisely the defect this accessor\n * was added to remove.\n *\n * Deliberately synchronous, total, and NOT `Result`-returning: it reads\n * constructor-injected configuration, touches no I/O, and cannot fail (every\n * link in the fallback chain has a total default). It exposes only the scope,\n * never the {@link IWritePolicy} itself, so it can never become a back door for\n * invoking admission or merge logic out of band.\n */\n dedupScopeFor(kind: Kind): DedupScope;\n\n /**\n * Whether records of `kind` participate in the **record-granular** vector index.\n *\n * @remarks\n * A read accessor over the store's injected {@link IFileTreeMemoryStoreCreateParams.embedKinds | embedKinds}\n * declaration, in the same spirit as {@link IMemoryStore.dedupScopeFor} — one\n * place to ask, so the store's write path and any caller reasoning about index\n * coverage cannot disagree. `true` for every kind when no declaration was made.\n *\n * **This is distinct from a {@link MemoryEmbedder} decline, and the difference is\n * cost.** An embedder that returns `undefined` has already been called: the\n * round trip is paid, and on a locally-hosted model that round trip is the\n * expense. A kind excluded here is never handed to the embedder at all. The\n * decline makes the intent *expressible*; this makes it *free*.\n *\n * Deliberately synchronous, total, and NOT `Result`-returning: it reads\n * constructor-injected configuration, touches no I/O, and cannot fail.\n */\n embedsKind(kind: Kind): boolean;\n\n /**\n * Write a record. Validates the body, computes a content hash, deduplicates\n * (scope-wide, before policy), applies the kind's {@link IWritePolicy}, stamps\n * transaction-time metadata (`created` / `updated` / `seq` / `contentHash`),\n * writes the file, and patches the index. Returns the written record — or the\n * existing record unchanged on a dedup no-op.\n */\n put(record: IMemoryRecord<unknown>): Promise<Result<IMemoryRecord<unknown>>>;\n\n /**\n * Repair one derived artifact for one {@link Kind} — **targeted and\n * non-destructive**, unlike `IVectorIndex.rebuild`, which resets the index and\n * re-embeds everything.\n *\n * @remarks\n * The counterpart to {@link IMemoryStore.coverage}: coverage says **how big**\n * the gap is and costs nothing; reconcile says **what it was** and closes what\n * it can, at the cost of reading bodies and re-running the embedder. Neither\n * substitutes for the other — in particular, only reconcile can distinguish a\n * *declined* record from a *failed* one, because learning that requires calling\n * the embedder again.\n *\n * **It only touches what is missing.** For the vector lanes it asks\n * `has(target)` per record and skips the ones already held, so a repair after a\n * brief outage costs a handful of embedder calls rather than a whole vault.\n * That check is also the only way to see a record whose vector the index holds\n * but whose envelope lost its `embeddingRef` — which needs a restamp and no\n * embedder call at all, and which an `embeddingRef`-only repair cannot detect.\n *\n * **`artifact` is required and names one lane.** See {@link DerivedArtifact}\n * for why an operation repairing \"everything wired\" would be the wrong shape.\n *\n * **A vector lane must be wired on BOTH halves — index *and* embedder — or this\n * fails, and it is deliberately stricter than {@link IMemoryStore.coverage}.**\n * The two ask different questions of the same wiring. Coverage asks *what does\n * the index hold*, which an index alone can answer: an index wired without an\n * embedder is a legal store (queries work; writes simply do not embed), so\n * coverage reports that lane rather than pretending it is absent. Reconcile\n * asks to *produce* vectors, which needs the embedder. So a half-wired store\n * legitimately reports a coverage gap it cannot repair, and reconcile names the\n * missing half rather than returning a cheerful success with every record in\n * `failed`.\n *\n * Runs under the store's write lock, like any other mutation.\n */\n reconcile(kind: Kind, artifact: DerivedArtifact): Promise<Result<ReconcileReport>>;\n\n /**\n * Delete a record by `(kind, entityId)`. Non-temporal kinds physically delete\n * the file and return the deleted record's {@link MemoryId}. Temporal\n * (versioned) kinds SOFT-delete: the current version is invalidated\n * (`invalid_at` set), history is retained, and the invalidated version's\n * {@link MemoryId} is returned.\n */\n delete(kind: Kind, entityId: EntityId): Promise<Result<MemoryId>>;\n}\n"]}
|
|
1
|
+
{"version":3,"file":"memoryStore.js","sourceRoot":"","sources":["../../../src/packlets/store/memoryStore.ts"],"names":[],"mappings":"AAAA;;;GAGG","sourcesContent":["/*\n * Copyright (c) 2026 Erik Fortune\n * SPDX-License-Identifier: MIT\n */\n\nimport { Result } from '@fgv/ts-utils';\nimport {\n DedupScope,\n EntityId,\n IMemoryRecord,\n IMemoryRecordResolver,\n Kind,\n MemoryId,\n MemoryScopeKey,\n IIdentityResolver\n} from '../types';\nimport { IIndexedMemoryEntry } from '../index';\nimport { IMemoryRecordSource, IScopedMemoryRecord } from '../vector';\nimport { MemoryListSelection } from './listSelection';\nimport { IDerivedStateCoverage } from './coverage';\nimport { DerivedArtifact, ReconcileReport } from './reconcile';\n\n/**\n * The writable, FileTree-backed, content-hash-deduped memory store.\n *\n * @remarks\n * Extends {@link IMemoryRecordResolver} because the retrievers take a resolver\n * and a store is the obvious one to hand them — `{ index, resolver: store }` is\n * the documented wiring, and it has to type-check for the `IMemoryStore` handle\n * a consumer actually holds, not only for the concrete `FileTreeMemoryStore`.\n * @public\n */\nexport interface IMemoryStore extends IMemoryRecordResolver, IIdentityResolver {\n /**\n * Keyed read by entity id. Resolves `entityId` to a storage address via the\n * registered {@link IIdentityCodec} for `kind`. Returns `undefined` when no\n * record exists. For a versioned (temporal) kind the current version is\n * *selected* from the derived in-memory index — the version history is walked\n * over envelopes, never bodies — and then that one version is read from\n * storage. So the selection costs no file reads and the result costs exactly\n * one, rather than one per version.\n */\n get(kind: Kind, entityId: EntityId): Promise<Result<IMemoryRecord<unknown> | undefined>>;\n\n /**\n * Direct read by `(scope, MemoryId)`. Returns `undefined` when not found.\n */\n getById(scope: MemoryScopeKey, id: MemoryId): Promise<Result<IMemoryRecord<unknown> | undefined>>;\n\n /**\n * List records: select over the derived index, then materialize the survivors.\n *\n * @remarks\n * **The selection is required and must narrow.** Since the index holds\n * envelopes only, every returned record is read from storage — so a call that\n * narrows nothing reads the whole vault, and that has to be a decision rather\n * than a default. Omitting the argument is a compile error; passing one with no\n * `scope` / `kind` / `tag` fails with a message naming {@link scanEveryRecord},\n * which is how a caller says it meant it.\n *\n * The requirement buys **explicitness, not a cost bound** — `{ kind }` on a\n * vault dominated by that kind still materializes most of it. What it prevents\n * is the whole-vault read nobody chose.\n *\n * If you only need to select, use {@link IMemoryStore.listEntries} instead: no\n * selection, no file reads.\n */\n list(selection: MemoryListSelection): Promise<Result<ReadonlyArray<IMemoryRecord<unknown>>>>;\n\n /**\n * How much of the store's **derived state** exists, resolved by {@link Kind}.\n *\n * @remarks\n * Answers *\"is my derived state consistent with my records, and if not by how\n * much?\"* for every artifact the store derives — `rank`, record vectors, and\n * fragment vectors — in one call.\n *\n * **Cheap and total, by contract rather than by implementation.** Every input is\n * an envelope field or an index-side count: it reads **no record bodies** and\n * calls **no embedder**, and the walk over the vault's own state touches the\n * filesystem not at all. That is why it takes no selection, unlike\n * {@link IMemoryStore.list} — the guard there exists because an unnarrowed list\n * reads the vault, and putting one here would decorate a free operation and make\n * that guard mean less.\n *\n * **The index-side counts are the one exception, and it is the caller's own\n * index that spends it.** A persistent index answers `size` / `recordCount` /\n * `fragmentCount` with a query — `SqliteVecVectorIndex` runs a prepared `COUNT`\n * — so on a durable backend this call does I/O and can **fail**. It is bounded\n * (one count per wired index, never per record) and it is why the return is a\n * `Result` rather than a bare value.\n *\n * *If a future addition to the report would require reading a record body, it\n * does not belong on this report.*\n *\n * The counterpart is {@link IMemoryStore.reconcile}: coverage says **how big**\n * the gap is, cheaply; reconcile says **what it was** and closes what it can, at\n * the cost of re-running the embedder. Neither substitutes for the other.\n *\n * **A lane is reported here whenever its *index* is wired**, which is weaker\n * than what reconcile requires (index **and** embedder). That is intended: an\n * index without an embedder still holds vectors and still answers queries, so\n * its coverage is a real number worth reporting. The consequence to expect is a\n * half-wired store that reports a gap `reconcile` will refuse to close, naming\n * the missing embedder.\n */\n coverage(): Promise<Result<IDerivedStateCoverage>>;\n\n /**\n * Every entry in the vault — scope and envelope, **no bodies**. Reads no files\n * and requires no selection, because there is nothing to be careful about: it\n * returns what the index already holds.\n *\n * @remarks\n * This is the whole-vault read most callers actually want. Selection, grouping,\n * counting, \"which kinds are in here\", \"what links at this\" — all of it is\n * envelope work. Reach for {@link IMemoryStore.list} with\n * {@link scanEveryRecord} only when you genuinely need every body.\n *\n * Synchronous in spirit but `Promise`-returning for consistency with the rest\n * of the store surface.\n */\n listEntries(): Promise<Result<ReadonlyArray<IIndexedMemoryEntry>>>;\n\n /**\n * List EVERY record in the vault, each paired with its scope-qualified\n * `(scope, id)` address — the projection {@link IMemoryRecordSource} requires.\n * Unlike {@link IMemoryStore.list | list}, it takes no filter (whole-vault) and\n * returns {@link IScopedMemoryRecord}s so a re-index keys each entry on the same\n * scoped target the incremental embed-on-write path uses. Two records that share\n * a filename stem across scopes appear as distinct entries.\n */\n listScoped(): Promise<Result<ReadonlyArray<IScopedMemoryRecord>>>;\n\n /**\n * Adapt this store to the {@link IMemoryRecordSource} seam so it can drive\n * {@link IVectorIndex} rebuilds (e.g. `InMemoryCosineIndex.rebuild`). The\n * returned source's `list()` delegates to {@link IMemoryStore.listScoped},\n * **filtered to the kinds {@link IMemoryStore.embedsKind | embedsKind} reports** —\n * this source exists to feed the record vector index, so a kind excluded from\n * that index is excluded here too, and a reopen does not re-embed records the\n * index will never return. With no\n * {@link IFileTreeMemoryStoreCreateParams.embedKinds | embedKinds} declaration\n * every kind participates and the filter is the identity. `listScoped` itself is\n * **not** filtered and remains the whole-vault surface.\n *\n * The filter also **counts what it drops**, onto\n * {@link IMemoryRecordListing.excluded} — this is the layer where the exclusion\n * decision is made and so the only one that can. The count is always present\n * (empty when nothing was excluded).\n *\n * The store cannot implement {@link IMemoryRecordSource} directly because its\n * `list(filter?)` returns bare records (the ergonomic query surface) while the\n * seam's `list()` returns scope-qualified records.\n */\n asRecordSource(): IMemoryRecordSource;\n\n /**\n * The EFFECTIVE {@link DedupScope} for `kind` — the granularity at which a\n * write for this kind deduplicates against the existing vault.\n *\n * @remarks\n * This is a **read accessor over the store's already-injected write policies**,\n * and it is the single place any caller — the store's own write path included —\n * asks what a kind's dedup granularity is. It resolves the full chain the store\n * applies on write: the kind's registered {@link IWritePolicy}, falling back to\n * the store's default policy, then that policy's\n * {@link IWritePolicy.dedupScope | dedupScope}, falling back to\n * {@link DEFAULT_DEDUP_SCOPE}. Note the store's default policy is a\n * {@link KnowledgeLwwPolicy}, which declares `'content'` — so a kind with NO\n * registered policy resolves to `'content'`, not to `DEFAULT_DEDUP_SCOPE`.\n *\n * It exists so a caller that must agree with the store about dedup granularity\n * — notably the ingest orchestrator's stage-4 layer-1 exact match — can read the\n * declaration through this seam instead of being handed a second copy of the\n * policy map. A second declaration site is precisely the defect this accessor\n * was added to remove.\n *\n * Deliberately synchronous, total, and NOT `Result`-returning: it reads\n * constructor-injected configuration, touches no I/O, and cannot fail (every\n * link in the fallback chain has a total default). It exposes only the scope,\n * never the {@link IWritePolicy} itself, so it can never become a back door for\n * invoking admission or merge logic out of band.\n */\n dedupScopeFor(kind: Kind): DedupScope;\n\n /**\n * Whether records of `kind` participate in the **record-granular** vector index.\n *\n * @remarks\n * A read accessor over the store's injected {@link IFileTreeMemoryStoreCreateParams.embedKinds | embedKinds}\n * declaration, in the same spirit as {@link IMemoryStore.dedupScopeFor} — one\n * place to ask, so the store's write path and any caller reasoning about index\n * coverage cannot disagree. `true` for every kind when no declaration was made.\n *\n * **This is distinct from a {@link MemoryEmbedder} decline, and the difference is\n * cost.** An embedder that returns `undefined` has already been called: the\n * round trip is paid, and on a locally-hosted model that round trip is the\n * expense. A kind excluded here is never handed to the embedder at all. The\n * decline makes the intent *expressible*; this makes it *free*.\n *\n * Deliberately synchronous, total, and NOT `Result`-returning: it reads\n * constructor-injected configuration, touches no I/O, and cannot fail.\n */\n embedsKind(kind: Kind): boolean;\n\n /**\n * Write a record. Validates the body, computes a content hash, deduplicates\n * (scope-wide, before policy), applies the kind's {@link IWritePolicy}, stamps\n * transaction-time metadata (`created` / `updated` / `seq` / `contentHash`),\n * writes the file, and patches the index. Returns the written record — or the\n * existing record unchanged on a dedup no-op.\n */\n put(record: IMemoryRecord<unknown>): Promise<Result<IMemoryRecord<unknown>>>;\n\n /**\n * Repair one derived artifact for one {@link Kind} — **targeted and\n * non-destructive**, unlike `IVectorIndex.rebuild`, which resets the index and\n * re-embeds everything.\n *\n * @remarks\n * The counterpart to {@link IMemoryStore.coverage}: coverage says **how big**\n * the gap is and costs nothing; reconcile says **what it was** and closes what\n * it can, at the cost of reading bodies and re-running the embedder. Neither\n * substitutes for the other — in particular, only reconcile can distinguish a\n * *declined* record from a *failed* one, because learning that requires calling\n * the embedder again.\n *\n * **It only touches what is missing.** For the vector lanes it asks\n * `has(target)` per record and skips the ones already held, so a repair after a\n * brief outage costs a handful of embedder calls rather than a whole vault.\n * That check is also the only way to see a record whose vector the index holds\n * but whose envelope lost its `embeddingRef` — which needs a restamp and no\n * embedder call at all, and which an `embeddingRef`-only repair cannot detect.\n *\n * **`artifact` is required and names one lane.** See {@link DerivedArtifact}\n * for why an operation repairing \"everything wired\" would be the wrong shape.\n *\n * **A vector lane must be wired on BOTH halves — index *and* embedder — or this\n * fails, and it is deliberately stricter than {@link IMemoryStore.coverage}.**\n * The two ask different questions of the same wiring. Coverage asks *what does\n * the index hold*, which an index alone can answer: an index wired without an\n * embedder is a legal store (queries work; writes simply do not embed), so\n * coverage reports that lane rather than pretending it is absent. Reconcile\n * asks to *produce* vectors, which needs the embedder. So a half-wired store\n * legitimately reports a coverage gap it cannot repair, and reconcile names the\n * missing half rather than returning a cheerful success with every record in\n * `failed`.\n *\n * Runs under the store's write lock, like any other mutation.\n */\n reconcile(kind: Kind, artifact: DerivedArtifact): Promise<Result<ReconcileReport>>;\n\n /**\n * Delete a record by `(kind, entityId)`. Non-temporal kinds physically delete\n * the file and return the deleted record's {@link MemoryId}. Temporal\n * (versioned) kinds SOFT-delete: the current version is invalidated\n * (`invalid_at` set), history is retained, and the invalidated version's\n * {@link MemoryId} is returned.\n */\n delete(kind: Kind, entityId: EntityId): Promise<Result<MemoryId>>;\n}\n"]}
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright (c) 2026 Erik Fortune
|
|
3
|
+
* SPDX-License-Identifier: MIT
|
|
4
|
+
*/
|
|
5
|
+
import { FileTree } from '@fgv/ts-json-base';
|
|
6
|
+
import { fail, succeed } from '@fgv/ts-utils';
|
|
7
|
+
/** The record filename extension; the store's files are markdown + YAML frontmatter. */
|
|
8
|
+
const MEMORY_FILE_EXTENSION = '.md';
|
|
9
|
+
/**
|
|
10
|
+
* The FileTree access layer beneath `FileTreeMemoryStore`: resolving and creating
|
|
11
|
+
* scope directories, and writing and deleting record files.
|
|
12
|
+
*
|
|
13
|
+
* @remarks
|
|
14
|
+
* Package-internal. Separated from the store because it is the one part of it that
|
|
15
|
+
* knows *where bytes live* rather than *what a record means* — it depends on nothing
|
|
16
|
+
* but the root directory and the scope encoder, and on no record semantics at all
|
|
17
|
+
* (no registry, no codec, no policy, no index). Extracted as free functions taking
|
|
18
|
+
* their two dependencies explicitly, matching `storeIdentity.ts`, so the seam is
|
|
19
|
+
* visible in the signatures rather than implied by `this`.
|
|
20
|
+
* @internal
|
|
21
|
+
*/
|
|
22
|
+
/**
|
|
23
|
+
* Resolve the directory for a scope, returning `undefined` when it does not
|
|
24
|
+
* exist. Navigation only — does not create. Folds the path segments through
|
|
25
|
+
* `getChildren` so an absent segment short-circuits to `undefined`.
|
|
26
|
+
*/
|
|
27
|
+
export function resolveScopeDir(root, scopeEncoding, scope) {
|
|
28
|
+
return scopeEncoding(scope).onSuccess((encoded) => {
|
|
29
|
+
const segments = encoded.split('/').filter((s) => s.length > 0);
|
|
30
|
+
return segments.reduce((acc, segment) => acc.onSuccess((current) => {
|
|
31
|
+
if (current === undefined) {
|
|
32
|
+
return succeed(undefined);
|
|
33
|
+
}
|
|
34
|
+
return current
|
|
35
|
+
.getChildren()
|
|
36
|
+
.onSuccess((children) => succeed(children.find((c) => c.type === 'directory' && c.name === segment)));
|
|
37
|
+
}), succeed(root));
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
/** Ensure the scope directory exists, creating segments as needed. */
|
|
41
|
+
export function ensureScopeDir(root, scopeEncoding, scope) {
|
|
42
|
+
return scopeEncoding(scope).onSuccess((encoded) => {
|
|
43
|
+
const segments = encoded.split('/').filter((s) => s.length > 0);
|
|
44
|
+
return segments.reduce((acc, segment) => acc.onSuccess((current) => current.getChildren().onSuccess((children) => {
|
|
45
|
+
const existing = children.find((c) => c.type === 'directory' && c.name === segment);
|
|
46
|
+
if (existing === undefined) {
|
|
47
|
+
return current.createChildDirectory(segment);
|
|
48
|
+
}
|
|
49
|
+
/* c8 ignore next 3 -- defensive: a child of a mutable in-memory/fs tree is itself mutable; the guard protects against a read-only adapter handed in as root */
|
|
50
|
+
if (!FileTree.isMutableDirectoryItem(existing)) {
|
|
51
|
+
return fail(`${existing.absolutePath}: directory is not mutable`);
|
|
52
|
+
}
|
|
53
|
+
return succeed(existing);
|
|
54
|
+
})), succeed(root));
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
/** Write (create or overwrite) `<scope>/<idStem>.md` with `raw`. */
|
|
58
|
+
export function writeRecordFile(root, scopeEncoding, scope, idStem, raw) {
|
|
59
|
+
return ensureScopeDir(root, scopeEncoding, scope).onSuccess((scopeDir) => scopeDir.getChildren().onSuccess((children) => {
|
|
60
|
+
const fileName = `${idStem}${MEMORY_FILE_EXTENSION}`;
|
|
61
|
+
const existing = children.find((c) => c.type === 'file' && c.name === fileName);
|
|
62
|
+
if (existing === undefined) {
|
|
63
|
+
return scopeDir.createChildFile(fileName, raw).onSuccess(() => succeed(true));
|
|
64
|
+
}
|
|
65
|
+
/* c8 ignore next 3 -- defensive: a file in a mutable tree is mutable; guards a read-only adapter */
|
|
66
|
+
if (!FileTree.isMutableFileItem(existing)) {
|
|
67
|
+
return fail(`${existing.absolutePath}: file is not mutable`);
|
|
68
|
+
}
|
|
69
|
+
return existing.setRawContents(raw).onSuccess(() => succeed(true));
|
|
70
|
+
}));
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Physically delete `<scope>/<idStem>.md`. The scope-missing and file-missing
|
|
74
|
+
* guards are unreachable through the callers (`delete` / `_evict` both read the
|
|
75
|
+
* record first, so the directory and file exist) but are kept so a future
|
|
76
|
+
* direct caller degrades loudly rather than silently.
|
|
77
|
+
*/
|
|
78
|
+
export function deleteRecordFile(root, scopeEncoding, scope, idStem) {
|
|
79
|
+
return resolveScopeDir(root, scopeEncoding, scope).onSuccess((scopeDir) => {
|
|
80
|
+
/* c8 ignore next 3 -- unreachable: callers read the record (hence the scope dir) first */
|
|
81
|
+
if (scopeDir === undefined) {
|
|
82
|
+
return fail(`memory delete: scope '${scope}' not found`);
|
|
83
|
+
}
|
|
84
|
+
const fileName = `${idStem}${MEMORY_FILE_EXTENSION}`;
|
|
85
|
+
return scopeDir.getChildren().onSuccess((children) => {
|
|
86
|
+
const file = children.find((c) => c.type === 'file' && c.name === fileName);
|
|
87
|
+
/* c8 ignore next 3 -- unreachable: callers read the record (hence the file) first */
|
|
88
|
+
if (file === undefined) {
|
|
89
|
+
return fail(`memory delete: file '${fileName}' not found in scope '${scope}'`);
|
|
90
|
+
}
|
|
91
|
+
/* c8 ignore next 3 -- defensive: a file in a mutable tree is mutable; guards a read-only adapter */
|
|
92
|
+
if (!FileTree.isMutableFileItem(file)) {
|
|
93
|
+
return fail(`${file.absolutePath}: file is not mutable`);
|
|
94
|
+
}
|
|
95
|
+
return file.delete().onSuccess(() => succeed(true));
|
|
96
|
+
});
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
//# sourceMappingURL=storeFileAccess.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"storeFileAccess.js","sourceRoot":"","sources":["../../../src/packlets/store/storeFileAccess.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC7C,OAAO,EAAU,IAAI,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAGtD,wFAAwF;AACxF,MAAM,qBAAqB,GAAW,KAAK,CAAC;AAS5C;;;;;;;;;;;;GAYG;AAEH;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAC7B,IAA4C,EAC5C,aAA2B,EAC3B,KAAqB;IAErB,OAAO,aAAa,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,EAAE;QAChD,MAAM,QAAQ,GAAa,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC1E,OAAO,QAAQ,CAAC,MAAM,CACpB,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CACf,GAAG,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,EAAE;YACxB,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC1B,OAAO,OAAO,CAAC,SAAS,CAAC,CAAC;YAC5B,CAAC;YACD,OAAO,OAAO;iBACX,WAAW,EAAE;iBACb,SAAS,CAAC,CAAC,QAAQ,EAAE,EAAE,CACtB,OAAO,CACL,QAAQ,CAAC,IAAI,CACX,CAAC,CAAC,EAAwC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO,CAC1F,CACF,CACF,CAAC;QACN,CAAC,CAAC,EACJ,OAAO,CAAC,IAAI,CAAC,CACd,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,sEAAsE;AACtE,MAAM,UAAU,cAAc,CAC5B,IAA4C,EAC5C,aAA2B,EAC3B,KAAqB;IAErB,OAAO,aAAa,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,EAAE;QAChD,MAAM,QAAQ,GAAa,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC1E,OAAO,QAAQ,CAAC,MAAM,CACpB,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CACf,GAAG,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,EAAE,CACxB,OAAO,CAAC,WAAW,EAAE,CAAC,SAAS,CAAC,CAAC,QAAQ,EAAE,EAAE;YAC3C,MAAM,QAAQ,GAAsC,QAAQ,CAAC,IAAI,CAC/D,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO,CACpD,CAAC;YACF,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;gBAC3B,OAAO,OAAO,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC;YAC/C,CAAC;YACD,+JAA+J;YAC/J,IAAI,CAAC,QAAQ,CAAC,sBAAsB,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC/C,OAAO,IAAI,CAAC,GAAG,QAAQ,CAAC,YAAY,4BAA4B,CAAC,CAAC;YACpE,CAAC;YACD,OAAO,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC3B,CAAC,CAAC,CACH,EACH,OAAO,CAAC,IAAI,CAAC,CACd,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,oEAAoE;AACpE,MAAM,UAAU,eAAe,CAC7B,IAA4C,EAC5C,aAA2B,EAC3B,KAAqB,EACrB,MAAc,EACd,GAAW;IAEX,OAAO,cAAc,CAAC,IAAI,EAAE,aAAa,EAAE,KAAK,CAAC,CAAC,SAAS,CAAC,CAAC,QAAQ,EAAE,EAAE,CACvE,QAAQ,CAAC,WAAW,EAAE,CAAC,SAAS,CAAC,CAAC,QAAQ,EAAE,EAAE;QAC5C,MAAM,QAAQ,GAAW,GAAG,MAAM,GAAG,qBAAqB,EAAE,CAAC;QAC7D,MAAM,QAAQ,GAAsC,QAAQ,CAAC,IAAI,CAC/D,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,IAAI,CAAC,CAAC,IAAI,KAAK,QAAQ,CAChD,CAAC;QACF,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,QAAQ,CAAC,eAAe,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;QAChF,CAAC;QACD,oGAAoG;QACpG,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC1C,OAAO,IAAI,CAAC,GAAG,QAAQ,CAAC,YAAY,uBAAuB,CAAC,CAAC;QAC/D,CAAC;QACD,OAAO,QAAQ,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;IACrE,CAAC,CAAC,CACH,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAC9B,IAA4C,EAC5C,aAA2B,EAC3B,KAAqB,EACrB,MAAc;IAEd,OAAO,eAAe,CAAC,IAAI,EAAE,aAAa,EAAE,KAAK,CAAC,CAAC,SAAS,CAAC,CAAC,QAAQ,EAAE,EAAE;QACxE,0FAA0F;QAC1F,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC,yBAAyB,KAAK,aAAa,CAAC,CAAC;QAC3D,CAAC;QACD,MAAM,QAAQ,GAAW,GAAG,MAAM,GAAG,qBAAqB,EAAE,CAAC;QAC7D,OAAO,QAAQ,CAAC,WAAW,EAAE,CAAC,SAAS,CAAC,CAAC,QAAQ,EAAE,EAAE;YACnD,MAAM,IAAI,GAAsC,QAAQ,CAAC,IAAI,CAC3D,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,IAAI,CAAC,CAAC,IAAI,KAAK,QAAQ,CAChD,CAAC;YACF,qFAAqF;YACrF,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;gBACvB,OAAO,IAAI,CAAC,wBAAwB,QAAQ,yBAAyB,KAAK,GAAG,CAAC,CAAC;YACjF,CAAC;YACD,oGAAoG;YACpG,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC,IAAI,CAAC,EAAE,CAAC;gBACtC,OAAO,IAAI,CAAC,GAAG,IAAI,CAAC,YAAY,uBAAuB,CAAC,CAAC;YAC3D,CAAC;YACD,OAAO,IAAI,CAAC,MAAM,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC","sourcesContent":["/*\n * Copyright (c) 2026 Erik Fortune\n * SPDX-License-Identifier: MIT\n */\n\nimport { FileTree } from '@fgv/ts-json-base';\nimport { Result, fail, succeed } from '@fgv/ts-utils';\nimport { MemoryScopeKey } from '../types';\n\n/** The record filename extension; the store's files are markdown + YAML frontmatter. */\nconst MEMORY_FILE_EXTENSION: string = '.md';\n\n/**\n * Maps a {@link MemoryScopeKey} to the relative directory path it is stored under.\n * Injected into the store, so this layer takes it rather than re-deriving it.\n * @internal\n */\nexport type ScopeEncoder = (scope: MemoryScopeKey) => Result<string>;\n\n/**\n * The FileTree access layer beneath `FileTreeMemoryStore`: resolving and creating\n * scope directories, and writing and deleting record files.\n *\n * @remarks\n * Package-internal. Separated from the store because it is the one part of it that\n * knows *where bytes live* rather than *what a record means* — it depends on nothing\n * but the root directory and the scope encoder, and on no record semantics at all\n * (no registry, no codec, no policy, no index). Extracted as free functions taking\n * their two dependencies explicitly, matching `storeIdentity.ts`, so the seam is\n * visible in the signatures rather than implied by `this`.\n * @internal\n */\n\n/**\n * Resolve the directory for a scope, returning `undefined` when it does not\n * exist. Navigation only — does not create. Folds the path segments through\n * `getChildren` so an absent segment short-circuits to `undefined`.\n */\nexport function resolveScopeDir(\n root: FileTree.IMutableFileTreeDirectoryItem,\n scopeEncoding: ScopeEncoder,\n scope: MemoryScopeKey\n): Result<FileTree.IFileTreeDirectoryItem | undefined> {\n return scopeEncoding(scope).onSuccess((encoded) => {\n const segments: string[] = encoded.split('/').filter((s) => s.length > 0);\n return segments.reduce<Result<FileTree.IFileTreeDirectoryItem | undefined>>(\n (acc, segment) =>\n acc.onSuccess((current) => {\n if (current === undefined) {\n return succeed(undefined);\n }\n return current\n .getChildren()\n .onSuccess((children) =>\n succeed(\n children.find(\n (c): c is FileTree.IFileTreeDirectoryItem => c.type === 'directory' && c.name === segment\n )\n )\n );\n }),\n succeed(root)\n );\n });\n}\n\n/** Ensure the scope directory exists, creating segments as needed. */\nexport function ensureScopeDir(\n root: FileTree.IMutableFileTreeDirectoryItem,\n scopeEncoding: ScopeEncoder,\n scope: MemoryScopeKey\n): Result<FileTree.IMutableFileTreeDirectoryItem> {\n return scopeEncoding(scope).onSuccess((encoded) => {\n const segments: string[] = encoded.split('/').filter((s) => s.length > 0);\n return segments.reduce<Result<FileTree.IMutableFileTreeDirectoryItem>>(\n (acc, segment) =>\n acc.onSuccess((current) =>\n current.getChildren().onSuccess((children) => {\n const existing: FileTree.FileTreeItem | undefined = children.find(\n (c) => c.type === 'directory' && c.name === segment\n );\n if (existing === undefined) {\n return current.createChildDirectory(segment);\n }\n /* c8 ignore next 3 -- defensive: a child of a mutable in-memory/fs tree is itself mutable; the guard protects against a read-only adapter handed in as root */\n if (!FileTree.isMutableDirectoryItem(existing)) {\n return fail(`${existing.absolutePath}: directory is not mutable`);\n }\n return succeed(existing);\n })\n ),\n succeed(root)\n );\n });\n}\n\n/** Write (create or overwrite) `<scope>/<idStem>.md` with `raw`. */\nexport function writeRecordFile(\n root: FileTree.IMutableFileTreeDirectoryItem,\n scopeEncoding: ScopeEncoder,\n scope: MemoryScopeKey,\n idStem: string,\n raw: string\n): Result<true> {\n return ensureScopeDir(root, scopeEncoding, scope).onSuccess((scopeDir) =>\n scopeDir.getChildren().onSuccess((children) => {\n const fileName: string = `${idStem}${MEMORY_FILE_EXTENSION}`;\n const existing: FileTree.FileTreeItem | undefined = children.find(\n (c) => c.type === 'file' && c.name === fileName\n );\n if (existing === undefined) {\n return scopeDir.createChildFile(fileName, raw).onSuccess(() => succeed(true));\n }\n /* c8 ignore next 3 -- defensive: a file in a mutable tree is mutable; guards a read-only adapter */\n if (!FileTree.isMutableFileItem(existing)) {\n return fail(`${existing.absolutePath}: file is not mutable`);\n }\n return existing.setRawContents(raw).onSuccess(() => succeed(true));\n })\n );\n}\n\n/**\n * Physically delete `<scope>/<idStem>.md`. The scope-missing and file-missing\n * guards are unreachable through the callers (`delete` / `_evict` both read the\n * record first, so the directory and file exist) but are kept so a future\n * direct caller degrades loudly rather than silently.\n */\nexport function deleteRecordFile(\n root: FileTree.IMutableFileTreeDirectoryItem,\n scopeEncoding: ScopeEncoder,\n scope: MemoryScopeKey,\n idStem: string\n): Result<true> {\n return resolveScopeDir(root, scopeEncoding, scope).onSuccess((scopeDir) => {\n /* c8 ignore next 3 -- unreachable: callers read the record (hence the scope dir) first */\n if (scopeDir === undefined) {\n return fail(`memory delete: scope '${scope}' not found`);\n }\n const fileName: string = `${idStem}${MEMORY_FILE_EXTENSION}`;\n return scopeDir.getChildren().onSuccess((children) => {\n const file: FileTree.FileTreeItem | undefined = children.find(\n (c) => c.type === 'file' && c.name === fileName\n );\n /* c8 ignore next 3 -- unreachable: callers read the record (hence the file) first */\n if (file === undefined) {\n return fail(`memory delete: file '${fileName}' not found in scope '${scope}'`);\n }\n /* c8 ignore next 3 -- defensive: a file in a mutable tree is mutable; guards a read-only adapter */\n if (!FileTree.isMutableFileItem(file)) {\n return fail(`${file.absolutePath}: file is not mutable`);\n }\n return file.delete().onSuccess(() => succeed(true));\n });\n });\n}\n"]}
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright (c) 2026 Erik Fortune
|
|
3
|
+
* SPDX-License-Identifier: MIT
|
|
4
|
+
*/
|
|
5
|
+
import { fail, succeed } from '@fgv/ts-utils';
|
|
6
|
+
/**
|
|
7
|
+
* The identity codec registered for `kind`, or the default.
|
|
8
|
+
*
|
|
9
|
+
* @remarks
|
|
10
|
+
* Package-internal. Extracted from `FileTreeMemoryStore` rather than living on it
|
|
11
|
+
* because the file is at its `max-lines` cap — the fourth consecutive stream to pay
|
|
12
|
+
* that toll, which `TECH_DEBT.md` names as the trigger to promote the split to P1.
|
|
13
|
+
* @internal
|
|
14
|
+
*/
|
|
15
|
+
export function codecFor(codecs, defaultCodec, kind) {
|
|
16
|
+
var _a;
|
|
17
|
+
const codec = (_a = codecs.get(kind)) !== null && _a !== void 0 ? _a : defaultCodec;
|
|
18
|
+
if (codec === undefined) {
|
|
19
|
+
return fail(`no identity codec registered for kind '${kind}'`);
|
|
20
|
+
}
|
|
21
|
+
return succeed(codec);
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Resolve `(kind, entityId)` to the storage address the vault files it under,
|
|
25
|
+
* without reading the record.
|
|
26
|
+
*
|
|
27
|
+
* @remarks
|
|
28
|
+
* `kind` selects the codec and the codec computes the address, so this is a
|
|
29
|
+
* function rather than a search — which is what makes an `EntityId` that collides
|
|
30
|
+
* across kinds a non-issue instead of an ambiguity to disambiguate.
|
|
31
|
+
* @internal
|
|
32
|
+
*/
|
|
33
|
+
export function resolveIdentity(codecs, defaultCodec, kind, entityId) {
|
|
34
|
+
return codecFor(codecs, defaultCodec, kind).onSuccess((codec) => codec.encode(entityId));
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Cross-check that a record read from a **kind-derived address** is of that kind.
|
|
38
|
+
*
|
|
39
|
+
* @remarks
|
|
40
|
+
* A record's address is `(scope, idStem)` and carries no kind component, so two
|
|
41
|
+
* kinds whose codecs can mint the same address name one file. Without this check
|
|
42
|
+
* the second write is not even an overwrite: the store reads the occupant, the
|
|
43
|
+
* policy merge rebuilds it as `{ ...existing.envelope, body: patched }`, and the
|
|
44
|
+
* victim keeps its own `kind` while taking the intruder's body — silently, because
|
|
45
|
+
* `kind` is immutable to every policy and so the write cannot look wrong. The
|
|
46
|
+
* intruder's own `list` then returns nothing.
|
|
47
|
+
*
|
|
48
|
+
* A kind's `IWritePolicy` cannot close this: its admission cohort is same-scope
|
|
49
|
+
* same-kind by contract, so a policy is never shown a foreign occupant. This is
|
|
50
|
+
* the only layer that sees both the incoming kind and the existing record.
|
|
51
|
+
*
|
|
52
|
+
* Pass `expected` only where a kind actually produced the address. An
|
|
53
|
+
* address-first read (`getById`) has no kind in play and must not be guarded.
|
|
54
|
+
* @internal
|
|
55
|
+
*/
|
|
56
|
+
export function verifyOccupantKind(expected, scope, idStem, record) {
|
|
57
|
+
if (expected !== undefined && record.envelope.kind !== expected) {
|
|
58
|
+
return fail(`memory address '${scope}/${idStem}' is occupied by a record of kind '${record.envelope.kind}', not '${expected}': two identity codecs mint the same address`);
|
|
59
|
+
}
|
|
60
|
+
return succeed(record);
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Cross-check a loaded record's declared identity against the address it was read
|
|
64
|
+
* from.
|
|
65
|
+
*
|
|
66
|
+
* @remarks
|
|
67
|
+
* The filename stem and the scope are the storage-side identity; the envelope's
|
|
68
|
+
* `id` / `entityId` are what downstream code trusts verbatim (merge-into
|
|
69
|
+
* re-addressing, for one). A tampered or corrupt file declaring a foreign
|
|
70
|
+
* `entityId` would otherwise load undetected, so the two are reconciled here
|
|
71
|
+
* through the codec's own round-trip.
|
|
72
|
+
* @internal
|
|
73
|
+
*/
|
|
74
|
+
export function verifyLoadedIdentity(codec, scope, file, record) {
|
|
75
|
+
if (record.envelope.id !== file.baseName) {
|
|
76
|
+
return fail(`memory file '${file.absolutePath}': envelope id '${record.envelope.id}' does not match filename stem '${file.baseName}'`);
|
|
77
|
+
}
|
|
78
|
+
return codec
|
|
79
|
+
.onSuccess((c) => c.verifyRoundTrip(scope, file.baseName).onSuccess(() => c.decode(scope, file.baseName)))
|
|
80
|
+
.withErrorFormat((msg) => `memory file '${file.absolutePath}': ${msg}`)
|
|
81
|
+
.onSuccess((decodedEntityId) => {
|
|
82
|
+
if (decodedEntityId !== record.envelope.entityId) {
|
|
83
|
+
return fail(`memory file '${file.absolutePath}': envelope entityId '${record.envelope.entityId}' does not match scope-derived entityId '${decodedEntityId}'`);
|
|
84
|
+
}
|
|
85
|
+
return succeed(record);
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
//# sourceMappingURL=storeIdentity.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"storeIdentity.js","sourceRoot":"","sources":["../../../src/packlets/store/storeIdentity.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAU,IAAI,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAUtD;;;;;;;;GAQG;AACH,MAAM,UAAU,QAAQ,CACtB,MAAyC,EACzC,YAAwC,EACxC,IAAU;;IAEV,MAAM,KAAK,GAA+B,MAAA,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,mCAAI,YAAY,CAAC;IAC3E,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,OAAO,IAAI,CAAC,0CAA0C,IAAI,GAAG,CAAC,CAAC;IACjE,CAAC;IACD,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC;AACxB,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,eAAe,CAC7B,MAAyC,EACzC,YAAwC,EACxC,IAAU,EACV,QAAkB;IAElB,OAAO,QAAQ,CAAC,MAAM,EAAE,YAAY,EAAE,IAAI,CAAC,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;AAC3F,CAAC;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,UAAU,kBAAkB,CAChC,QAA0B,EAC1B,KAAqB,EACrB,MAAc,EACd,MAA0D;IAE1D,IAAI,QAAQ,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAChE,OAAO,IAAI,CACT,mBAAmB,KAAK,IAAI,MAAM,sCAAsC,MAAM,CAAC,QAAQ,CAAC,IAAI,WAAW,QAAQ,8CAA8C,CAC9J,CAAC;IACJ,CAAC;IACD,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC;AACzB,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,oBAAoB,CAClC,KAA6B,EAC7B,KAAqB,EACrB,IAAgC,EAChC,MAA8B;IAE9B,IAAI,MAAM,CAAC,QAAQ,CAAC,EAAE,KAAK,IAAI,CAAC,QAAQ,EAAE,CAAC;QACzC,OAAO,IAAI,CACT,gBAAgB,IAAI,CAAC,YAAY,mBAAmB,MAAM,CAAC,QAAQ,CAAC,EAAE,mCAAmC,IAAI,CAAC,QAAQ,GAAG,CAC1H,CAAC;IACJ,CAAC;IACD,OAAO,KAAK;SACT,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;SACzG,eAAe,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,gBAAgB,IAAI,CAAC,YAAY,MAAM,GAAG,EAAE,CAAC;SACtE,SAAS,CAAC,CAAC,eAAe,EAAE,EAAE;QAC7B,IAAI,eAAe,KAAK,MAAM,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC;YACjD,OAAO,IAAI,CACT,gBAAgB,IAAI,CAAC,YAAY,yBAAyB,MAAM,CAAC,QAAQ,CAAC,QAAQ,4CAA4C,eAAe,GAAG,CACjJ,CAAC;QACJ,CAAC;QACD,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC;IACzB,CAAC,CAAC,CAAC;AACP,CAAC","sourcesContent":["/*\n * Copyright (c) 2026 Erik Fortune\n * SPDX-License-Identifier: MIT\n */\n\nimport { FileTree } from '@fgv/ts-json-base';\nimport { Result, fail, succeed } from '@fgv/ts-utils';\nimport {\n EntityId,\n IIdentityCodec,\n IIdentityCodecResult,\n IMemoryRecord,\n Kind,\n MemoryScopeKey\n} from '../types';\n\n/**\n * The identity codec registered for `kind`, or the default.\n *\n * @remarks\n * Package-internal. Extracted from `FileTreeMemoryStore` rather than living on it\n * because the file is at its `max-lines` cap — the fourth consecutive stream to pay\n * that toll, which `TECH_DEBT.md` names as the trigger to promote the split to P1.\n * @internal\n */\nexport function codecFor(\n codecs: ReadonlyMap<Kind, IIdentityCodec>,\n defaultCodec: IIdentityCodec | undefined,\n kind: Kind\n): Result<IIdentityCodec> {\n const codec: IIdentityCodec | undefined = codecs.get(kind) ?? defaultCodec;\n if (codec === undefined) {\n return fail(`no identity codec registered for kind '${kind}'`);\n }\n return succeed(codec);\n}\n\n/**\n * Resolve `(kind, entityId)` to the storage address the vault files it under,\n * without reading the record.\n *\n * @remarks\n * `kind` selects the codec and the codec computes the address, so this is a\n * function rather than a search — which is what makes an `EntityId` that collides\n * across kinds a non-issue instead of an ambiguity to disambiguate.\n * @internal\n */\nexport function resolveIdentity(\n codecs: ReadonlyMap<Kind, IIdentityCodec>,\n defaultCodec: IIdentityCodec | undefined,\n kind: Kind,\n entityId: EntityId\n): Result<IIdentityCodecResult> {\n return codecFor(codecs, defaultCodec, kind).onSuccess((codec) => codec.encode(entityId));\n}\n\n/**\n * Cross-check that a record read from a **kind-derived address** is of that kind.\n *\n * @remarks\n * A record's address is `(scope, idStem)` and carries no kind component, so two\n * kinds whose codecs can mint the same address name one file. Without this check\n * the second write is not even an overwrite: the store reads the occupant, the\n * policy merge rebuilds it as `{ ...existing.envelope, body: patched }`, and the\n * victim keeps its own `kind` while taking the intruder's body — silently, because\n * `kind` is immutable to every policy and so the write cannot look wrong. The\n * intruder's own `list` then returns nothing.\n *\n * A kind's `IWritePolicy` cannot close this: its admission cohort is same-scope\n * same-kind by contract, so a policy is never shown a foreign occupant. This is\n * the only layer that sees both the incoming kind and the existing record.\n *\n * Pass `expected` only where a kind actually produced the address. An\n * address-first read (`getById`) has no kind in play and must not be guarded.\n * @internal\n */\nexport function verifyOccupantKind<T>(\n expected: Kind | undefined,\n scope: MemoryScopeKey,\n idStem: string,\n record: T & { readonly envelope: { readonly kind: Kind } }\n): Result<T> {\n if (expected !== undefined && record.envelope.kind !== expected) {\n return fail(\n `memory address '${scope}/${idStem}' is occupied by a record of kind '${record.envelope.kind}', not '${expected}': two identity codecs mint the same address`\n );\n }\n return succeed(record);\n}\n\n/**\n * Cross-check a loaded record's declared identity against the address it was read\n * from.\n *\n * @remarks\n * The filename stem and the scope are the storage-side identity; the envelope's\n * `id` / `entityId` are what downstream code trusts verbatim (merge-into\n * re-addressing, for one). A tampered or corrupt file declaring a foreign\n * `entityId` would otherwise load undetected, so the two are reconciled here\n * through the codec's own round-trip.\n * @internal\n */\nexport function verifyLoadedIdentity(\n codec: Result<IIdentityCodec>,\n scope: MemoryScopeKey,\n file: FileTree.IFileTreeFileItem,\n record: IMemoryRecord<unknown>\n): Result<IMemoryRecord<unknown>> {\n if (record.envelope.id !== file.baseName) {\n return fail(\n `memory file '${file.absolutePath}': envelope id '${record.envelope.id}' does not match filename stem '${file.baseName}'`\n );\n }\n return codec\n .onSuccess((c) => c.verifyRoundTrip(scope, file.baseName).onSuccess(() => c.decode(scope, file.baseName)))\n .withErrorFormat((msg) => `memory file '${file.absolutePath}': ${msg}`)\n .onSuccess((decodedEntityId) => {\n if (decodedEntityId !== record.envelope.entityId) {\n return fail(\n `memory file '${file.absolutePath}': envelope entityId '${record.envelope.entityId}' does not match scope-derived entityId '${decodedEntityId}'`\n );\n }\n return succeed(record);\n });\n}\n"]}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"identityResolver.js","sourceRoot":"","sources":["../../../src/packlets/types/identityResolver.ts"],"names":[],"mappings":"AAAA;;;GAGG","sourcesContent":["/*\n * Copyright (c) 2026 Erik Fortune\n * SPDX-License-Identifier: MIT\n */\n\nimport { Result } from '@fgv/ts-utils';\nimport { IIdentityCodecResult } from './identityCodec';\nimport { EntityId, Kind } from './ids';\n\n/**\n * Resolves a consumer-facing `(kind, entityId)` address to the storage address the\n * vault files it under, **without reading the record**.\n *\n * @remarks\n * This is the resolution `IMemoryStore.get(kind, entityId)` already performs before\n * it reads anything: `kind` selects the kind's `IIdentityCodec`, and the codec's\n * `encode` computes `{ scope, idStem, isVersioned }`. Exposing it separately lets a\n * caller that needs only the *address* — a fragment query narrowing to one record,\n * say — avoid paying for a read it does not want.\n *\n * **The resolution is a function, not a search, and that is the load-bearing\n * property.** A consumer holds an `EntityId`; a vector hit is addressed by a\n * `(scope, id)` pair; and `EntityId` promises no uniqueness beyond a scope, so the\n * same id may legitimately appear in several scopes (a document `acme-corp` under\n * one kind and the entity `acme-corp` under another is the ordinary case, not a\n * pathological one). Supplying `kind` selects one codec, and a codec cannot return\n * two answers — so ambiguity is structurally impossible rather than merely unlikely,\n * and no disambiguation pass is needed anywhere downstream.\n *\n * Narrow by design, mirroring `IMemoryRecordResolver`: a component that needs to\n * turn an entity address into a storage address should depend on this rather than on\n * the whole store.\n * @public\n */\nexport interface IIdentityResolver {\n /**\n * The storage address `(kind, entityId)` maps to, without reading the record.\n *\n * @param kind - Selects the identity codec. Required: it is what makes the\n * resolution unambiguous.\n * @param entityId - The consumer-supplied domain key.\n * @returns `Success` with the codec's `{ scope, idStem, isVersioned }`, or\n * `Failure` if no codec is registered for `kind` (and no default is wired) or the\n * codec rejects the id. Both are caller errors and both are loud.\n */\n resolveIdentity(kind: Kind, entityId: EntityId): Result<IIdentityCodecResult>;\n}\n"]}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/packlets/types/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,cAAc,OAAO,CAAC;AACtB,cAAc,YAAY,CAAC;AAC3B,cAAc,kBAAkB,CAAC;AACjC,cAAc,iBAAiB,CAAC;AAChC,cAAc,kBAAkB,CAAC;AACjC,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC","sourcesContent":["/*\n * Copyright (c) 2026 Erik Fortune\n * SPDX-License-Identifier: MIT\n */\n\nexport * from './ids';\nexport * from './envelope';\nexport * from './filenameSafety';\nexport * from './identityCodec';\nexport * from './recordResolver';\nexport * from './temporal';\nexport * from './writePolicy';\n"]}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/packlets/types/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,cAAc,OAAO,CAAC;AACtB,cAAc,YAAY,CAAC;AAC3B,cAAc,kBAAkB,CAAC;AACjC,cAAc,iBAAiB,CAAC;AAChC,cAAc,oBAAoB,CAAC;AACnC,cAAc,kBAAkB,CAAC;AACjC,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC","sourcesContent":["/*\n * Copyright (c) 2026 Erik Fortune\n * SPDX-License-Identifier: MIT\n */\n\nexport * from './ids';\nexport * from './envelope';\nexport * from './filenameSafety';\nexport * from './identityCodec';\nexport * from './identityResolver';\nexport * from './recordResolver';\nexport * from './temporal';\nexport * from './writePolicy';\n"]}
|
|
@@ -107,8 +107,9 @@ export class InMemoryFragmentCosineIndex {
|
|
|
107
107
|
return Promise.resolve(succeed(target));
|
|
108
108
|
}
|
|
109
109
|
/** {@inheritDoc IFragmentVectorIndex.query} */
|
|
110
|
-
query(vector, topK,
|
|
110
|
+
query(vector, topK, options) {
|
|
111
111
|
var _a;
|
|
112
|
+
const maxPerRecord = options === null || options === void 0 ? void 0 : options.maxPerRecord;
|
|
112
113
|
if (topK <= 0 || this._records.size === 0) {
|
|
113
114
|
return Promise.resolve(succeed([]));
|
|
114
115
|
}
|
|
@@ -117,7 +118,12 @@ export class InMemoryFragmentCosineIndex {
|
|
|
117
118
|
}
|
|
118
119
|
const queryMagnitude = InMemoryFragmentCosineIndex._magnitude(vector);
|
|
119
120
|
const scored = [];
|
|
120
|
-
|
|
121
|
+
// The narrowing is applied HERE — choosing which records are scored at all —
|
|
122
|
+
// rather than by filtering hits afterwards. That is the whole point: a
|
|
123
|
+
// post-filter would leave `topK` applied to the global set, so a scoped search
|
|
124
|
+
// would silently return fewer than `topK` whenever other records outscored the
|
|
125
|
+
// target's fragments.
|
|
126
|
+
for (const record of this._selectRecords(options)) {
|
|
121
127
|
for (const fragment of record.fragments) {
|
|
122
128
|
scored.push({
|
|
123
129
|
key: edgeTargetKey(record.target),
|
|
@@ -149,24 +155,39 @@ export class InMemoryFragmentCosineIndex {
|
|
|
149
155
|
return Promise.resolve(succeed(hits));
|
|
150
156
|
}
|
|
151
157
|
/**
|
|
152
|
-
*
|
|
153
|
-
* scratch. Clears the current contents (and the established dimension) first, so
|
|
154
|
-
* a re-embed with a different model is supported. Returns the total number of
|
|
155
|
-
* fragments indexed.
|
|
156
|
-
*
|
|
157
|
-
* On any failure (list, embed, or add) the index is rolled back to empty rather
|
|
158
|
-
* than left in a partially-rebuilt state.
|
|
158
|
+
* The records a query is allowed to score, honoring the `scope` / `id` narrowing.
|
|
159
159
|
*
|
|
160
160
|
* @remarks
|
|
161
|
-
*
|
|
162
|
-
*
|
|
163
|
-
*
|
|
164
|
-
*
|
|
165
|
-
*
|
|
166
|
-
*
|
|
167
|
-
* @param source - The scope-qualified record source to re-embed.
|
|
168
|
-
* @param embed - The fragment embedder applied to each record.
|
|
161
|
+
* The single-record case is an O(1) map lookup rather than a scan, because the
|
|
162
|
+
* record map is keyed by `edgeTargetKey`. The scope-only case (a versioned kind's
|
|
163
|
+
* per-entity subtree) is a filtered walk — still bounded by the vault, but it
|
|
164
|
+
* scores only the entity's own fragments, which is what makes the caller's `topK`
|
|
165
|
+
* meaningful.
|
|
169
166
|
*/
|
|
167
|
+
*_selectRecords(options) {
|
|
168
|
+
// Narrowed once, deliberately: re-deriving through `options?.` a second time
|
|
169
|
+
// after an early return that already implies `options !== undefined` creates an
|
|
170
|
+
// optional-chain arm that cannot fire, which is a dead branch rather than an
|
|
171
|
+
// untested one.
|
|
172
|
+
if (options === undefined || options.scope === undefined) {
|
|
173
|
+
yield* this._records.values();
|
|
174
|
+
return;
|
|
175
|
+
}
|
|
176
|
+
const scope = options.scope;
|
|
177
|
+
const id = options.id;
|
|
178
|
+
if (id !== undefined) {
|
|
179
|
+
const record = this._records.get(edgeTargetKey({ scope, id }));
|
|
180
|
+
if (record !== undefined) {
|
|
181
|
+
yield record;
|
|
182
|
+
}
|
|
183
|
+
return;
|
|
184
|
+
}
|
|
185
|
+
for (const record of this._records.values()) {
|
|
186
|
+
if (record.target.scope === scope) {
|
|
187
|
+
yield record;
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
}
|
|
170
191
|
/** {@inheritDoc IFragmentVectorIndex.rebuild} */
|
|
171
192
|
async rebuild(source, embed, options) {
|
|
172
193
|
var _a;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"inMemoryFragmentCosineIndex.js","sourceRoot":"","sources":["../../../src/packlets/vector/inMemoryFragmentCosineIndex.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAA0B,IAAI,EAAE,cAAc,EAAE,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AACzG,OAAO,EAAqB,aAAa,EAAE,MAAM,UAAU,CAAC;AAY5D,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAUrD,0FAA0F;AAC1F,SAAS,gBAAgB,CAAC,QAA2B;IACnD,uCACK,CAAC,QAAQ,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,GACrE,CAAC,QAAQ,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EACjF;AACJ,CAAC;AAoBD;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,OAAO,2BAA2B;IAUtC;QACE,IAAI,CAAC,QAAQ,GAAG,IAAI,GAAG,EAAkC,CAAC;QAC1D,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;IAC9B,CAAC;IAED,8EAA8E;IAC9E,IAAW,WAAW;QACpB,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;IAC5B,CAAC;IAED,uEAAuE;IACvE,IAAW,aAAa;QACtB,IAAI,KAAK,GAAW,CAAC,CAAC;QACtB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;YAC5C,KAAK,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC;QACnC,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,iCAAiC;IAC1B,MAAM,CAAC,MAAM;QAClB,OAAO,OAAO,CAAC,IAAI,2BAA2B,EAAE,CAAC,CAAC;IACpD,CAAC;IAED,sDAAsD;IAC/C,YAAY,CACjB,MAAmB,EACnB,SAA2C;QAE3C,MAAM,GAAG,GAAW,aAAa,CAAC,MAAM,CAAC,CAAC;QAC1C,6EAA6E;QAC7E,0EAA0E;QAC1E,gFAAgF;QAChF,8EAA8E;QAC9E,iFAAiF;QACjF,MAAM,MAAM,GAAsB,EAAE,CAAC;QACrC,IAAI,SAAS,GAAuB,IAAI,CAAC,UAAU,CAAC;QACpD,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,IAAI,QAAQ,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACjC,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,+BAA+B,GAAG,0BAA0B,CAAC,CAAC,CAAC;YAC7F,CAAC;YACD,gFAAgF;YAChF,4EAA4E;YAC5E,uEAAuE;YACvE,IAAI,QAAQ,CAAC,OAAO,KAAK,SAAS,IAAI,QAAQ,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;gBACxE,OAAO,OAAO,CAAC,OAAO,CACpB,IAAI,CACF,+BAA+B,GAAG,gEAAgE,CACnG,CACF,CAAC;YACJ,CAAC;YACD,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;gBAC5B,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC;YACrC,CAAC;iBAAM,IAAI,QAAQ,CAAC,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;gBAChD,OAAO,OAAO,CAAC,OAAO,CACpB,IAAI,CACF,+BAA+B,GAAG,yBAAyB,QAAQ,CAAC,MAAM,CAAC,MAAM,mCAAmC,SAAS,EAAE,CAChI,CACF,CAAC;YACJ,CAAC;YACD,kFAAkF;YAClF,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,gBAAgB,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACpG,CAAC;QACD,6EAA6E;QAC7E,iFAAiF;QACjF,0DAA0D;QAC1D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC5B,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;YAC5B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;QACxD,CAAC;QACD,OAAO,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IACjD,CAAC;IAED,6CAA6C;IACtC,GAAG,CAAC,MAAmB;QAC5B,OAAO,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5E,CAAC;IAED,gDAAgD;IACzC,MAAM,CAAC,MAAmB;QAC/B,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC;QAC5C,OAAO,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;IAC1C,CAAC;IAED,+CAA+C;IACxC,KAAK,CACV,MAAoB,EACpB,IAAY,EACZ,YAAqB;;QAErB,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YAC1C,OAAO,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;QACtC,CAAC;QACD,IAAI,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC;YACtC,OAAO,OAAO,CAAC,OAAO,CACpB,IAAI,CACF,mCAAmC,MAAM,CAAC,MAAM,mCAAmC,IAAI,CAAC,UAAU,EAAE,CACrG,CACF,CAAC;QACJ,CAAC;QACD,MAAM,cAAc,GAAW,2BAA2B,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAC9E,MAAM,MAAM,GAAsB,EAAE,CAAC;QACrC,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;YAC5C,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;gBACxC,MAAM,CAAC,IAAI,CAAC;oBACV,GAAG,EAAE,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC;oBACjC,GAAG,kBACD,MAAM,EAAE,MAAM,CAAC,MAAM,EACrB,KAAK,EAAE,2BAA2B,CAAC,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,QAAQ,CAAC,MAAM,CAAC,IAChF,QAAQ,CAAC,QAAQ,CACrB;iBACF,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,4EAA4E;QAC5E,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAEjD,MAAM,IAAI,GAAsB,EAAE,CAAC;QACnC,8EAA8E;QAC9E,6EAA6E;QAC7E,4EAA4E;QAC5E,wDAAwD;QACxD,MAAM,SAAS,GAAwB,IAAI,GAAG,EAAkB,CAAC;QACjE,KAAK,MAAM,SAAS,IAAI,MAAM,EAAE,CAAC;YAC/B,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,EAAE,CAAC;gBACxB,MAAM;YACR,CAAC;YACD,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;gBAC/B,MAAM,IAAI,GAAW,MAAA,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,mCAAI,CAAC,CAAC;gBACvD,IAAI,IAAI,IAAI,YAAY,EAAE,CAAC;oBACzB,SAAS;gBACX,CAAC;gBACD,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC;YACzC,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAC3B,CAAC;QACD,OAAO,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;IACxC,CAAC;IAED;;;;;;;;;;;;;;;;;;OAkBG;IACH,iDAAiD;IAC1C,KAAK,CAAC,OAAO,CAClB,MAA2B,EAC3B,KAAuB,EACvB,OAA+B;;QAE/B,MAAM,OAAO,GAAY,CAAC,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,aAAa,mCAAI,MAAM,CAAC,KAAK,MAAM,CAAC;QACvE,MAAM,MAAM,GAAiC,MAAM,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;QACnF,IAAI,MAAM,CAAC,SAAS,EAAE,EAAE,CAAC;YACvB,yEAAyE;YACzE,2EAA2E;YAC3E,uEAAuE;YACvE,0EAA0E;YAC1E,0EAA0E;YAC1E,oCAAoC;YACpC,OAAO,cAAc,CAAC,mDAAmD,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;QAC7F,CAAC;QACD,0EAA0E;QAC1E,yEAAyE;QACzE,IAAI,CAAC,MAAM,EAAE,CAAC;QACd,MAAM,OAAO,GAAsB,IAAI,GAAG,EAAgB,CAAC;QAC3D,MAAM,SAAS,GAAsB,IAAI,GAAG,EAAgB,CAAC;QAC7D,MAAM,QAAQ,GAAsB,IAAI,GAAG,EAAgB,CAAC;QAC5D,MAAM,OAAO,GAA2B,EAAE,CAAC;QAC3C,6EAA6E;QAC7E,4EAA4E;QAC5E,8EAA8E;QAC9E,UAAU;QACV,MAAM,MAAM,GAAG,GAAiC,EAAE,CAAC,CAAC;YAClD,OAAO;YACP,SAAS;YACT,QAAQ;YACR,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ;YAC/B,OAAO;SACR,CAAC,CAAC;QACH,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;YAC1C,MAAM,IAAI,GAAS,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC;YAC/C,qEAAqE;YACrE,0DAA0D;YAC1D,MAAM,QAAQ,GAA6C,MAAM,UAAU,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;YACxG,IAAI,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC;gBACzB,MAAM,KAAK,GAAW,sCAAsC,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,aACtF,QAAQ,CAAC,OACX,EAAE,CAAC;gBACH,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,IAAI,CAAC,MAAM,EAAE,CAAC;oBACd,OAAO,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;gBACzC,CAAC;gBACD,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;gBAC/C,SAAS;YACX,CAAC;YACD,kEAAkE;YAClE,0EAA0E;YAC1E,4DAA4D;YAC5D,MAAM,KAAK,GAAmB,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;YACrF,IAAI,KAAK,CAAC,SAAS,EAAE,EAAE,CAAC;gBACtB,MAAM,KAAK,GAAW,2BAA2B,KAAK,CAAC,OAAO,EAAE,CAAC;gBACjE,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,IAAI,CAAC,MAAM,EAAE,CAAC;oBACd,OAAO,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;gBACzC,CAAC;gBACD,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;gBAC/C,SAAS;YACX,CAAC;YACD,IAAI,KAAK,CAAC,KAAK,KAAK,CAAC,EAAE,CAAC;gBACtB,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;gBACtB,SAAS;YACX,CAAC;YACD,yEAAyE;YACzE,4DAA4D;YAC5D,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;YACrB,KAAK,CAAC,SAAS,EAAE,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;QACtC,CAAC;QACD,OAAO,iBAAiB,CAAC,MAAM,EAAE,CAAC,CAAC;IACrC,CAAC;IAED,4DAA4D;IACpD,MAAM;QACZ,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;IAC9B,CAAC;IAED,qDAAqD;IAC7C,MAAM,CAAC,UAAU,CAAC,MAAoB;QAC5C,IAAI,GAAG,GAAW,CAAC,CAAC;QACpB,KAAK,IAAI,CAAC,GAAW,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/C,GAAG,IAAI,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAC/B,CAAC;QACD,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACxB,CAAC;IAED;;;;OAIG;IACK,MAAM,CAAC,OAAO,CAAC,KAAmB,EAAE,cAAsB,EAAE,MAAoB;QACtF,MAAM,eAAe,GAAW,2BAA2B,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAC/E,IAAI,cAAc,KAAK,CAAC,IAAI,eAAe,KAAK,CAAC,EAAE,CAAC;YAClD,OAAO,CAAC,CAAC;QACX,CAAC;QACD,IAAI,GAAG,GAAW,CAAC,CAAC;QACpB,KAAK,IAAI,CAAC,GAAW,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC9C,GAAG,IAAI,KAAK,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAC9B,CAAC;QACD,OAAO,GAAG,GAAG,CAAC,cAAc,GAAG,eAAe,CAAC,CAAC;IAClD,CAAC;CACF","sourcesContent":["/*\n * Copyright (c) 2026 Erik Fortune\n * SPDX-License-Identifier: MIT\n */\n\nimport { DetailedResult, Result, fail, failWithDetail, succeed, succeedWithDetail } from '@fgv/ts-utils';\nimport { IEdgeTarget, Kind, edgeTargetKey } from '../types';\nimport {\n FragmentEmbedder,\n IEmbeddedFragment,\n IFragmentVectorIndex,\n IFragmentVectorRebuildReport,\n IMemoryRecordListing,\n IMemoryRecordSource,\n ISkippedVectorRecord,\n IVectorQueryHit,\n IVectorRebuildOptions\n} from './vectorIndex';\nimport { invokeHook, tally } from './rebuildHelpers';\n\n/**\n * The identity fields a fragment was added with, already in query-hit shape: a field\n * the fragment did not carry is *absent*, never present-but-`undefined`, so a hit\n * for a fragment added without a `fragmentId` is structurally identical to one\n * produced before `fragmentId` existed.\n */\ntype FragmentIdentity = Pick<IVectorQueryHit, 'locator' | 'fragmentId'>;\n\n/** Project an incoming fragment's identity fields, dropping the ones it did not carry. */\nfunction fragmentIdentity(fragment: IEmbeddedFragment): FragmentIdentity {\n return {\n ...(fragment.locator !== undefined ? { locator: fragment.locator } : {}),\n ...(fragment.fragmentId !== undefined ? { fragmentId: fragment.fragmentId } : {})\n };\n}\n\n/** One stored fragment: the identity it was added with plus its vector. */\ninterface IStoredFragment {\n readonly identity: FragmentIdentity;\n readonly vector: Float32Array;\n}\n\n/** Every stored fragment for one record, tagged with the record's scoped address. */\ninterface IStoredRecordFragments {\n readonly target: IEdgeTarget;\n readonly fragments: IStoredFragment[];\n}\n\n/** A candidate hit carried through selection: the fragment's key, hit, and score. */\ninterface IScoredFragment {\n readonly key: string;\n readonly hit: IVectorQueryHit;\n}\n\n/**\n * The brute-force, in-memory cosine {@link IFragmentVectorIndex} — the\n * fragment-granular sibling of {@link InMemoryCosineIndex}. Stores many\n * `Float32Array`s per record (one per {@link IEmbeddedFragment | fragment}) and\n * answers a query by computing cosine similarity against every stored fragment,\n * returning the top-k fragment hits by descending score. Each hit carries back\n * whichever of {@link IFragmentLocator | locator} / `fragmentId` its fragment was\n * added with; a fragment must carry at least one of the two.\n *\n * @remarks\n * Same regime and same non-goals as {@link InMemoryCosineIndex}: no external\n * dependency, no ANN structure, a linear scan over the stored fragments — the seam\n * ({@link IFragmentVectorIndex}) stays open for a consumer to swap a persistent /\n * ANN backend once N grows. `addFragments` is whole-record-replace, so re-authoring\n * a document never leaves stale fragments behind. The index has a single dimension\n * established by the first fragment added; every subsequent fragment and every\n * `query` vector must match it or fail loudly — a mismatched dimension is an\n * embedder-wiring bug, never a silent zero-similarity result.\n * {@link InMemoryFragmentCosineIndex.rebuild | rebuild} clears the index (and the\n * established dimension), so a re-embed with a different model is supported.\n *\n * The optional `maxPerRecord` cap on `query` is applied **during selection**, before\n * the `topK` cut, so one long document with many strong fragments cannot crowd every\n * other record out of the result.\n * @public\n */\nexport class InMemoryFragmentCosineIndex implements IFragmentVectorIndex {\n /**\n * Stored fragments keyed by the canonical {@link edgeTargetKey} of the record's\n * scope-qualified address, so two records that share a filename stem across\n * scopes occupy distinct entries and never overwrite each other's fragments.\n */\n private readonly _records: Map<string, IStoredRecordFragments>;\n /** The dimension of every stored fragment vector; `undefined` until the first `add`. */\n private _dimension: number | undefined;\n\n private constructor() {\n this._records = new Map<string, IStoredRecordFragments>();\n this._dimension = undefined;\n }\n\n /** The number of records that currently have at least one stored fragment. */\n public get recordCount(): number {\n return this._records.size;\n }\n\n /** The total number of fragments currently held across all records. */\n public get fragmentCount(): number {\n let total: number = 0;\n for (const record of this._records.values()) {\n total += record.fragments.length;\n }\n return total;\n }\n\n /** Family-convention factory. */\n public static create(): Result<InMemoryFragmentCosineIndex> {\n return succeed(new InMemoryFragmentCosineIndex());\n }\n\n /** {@inheritDoc IFragmentVectorIndex.addFragments} */\n public addFragments(\n target: IEdgeTarget,\n fragments: ReadonlyArray<IEmbeddedFragment>\n ): Promise<Result<number>> {\n const key: string = edgeTargetKey(target);\n // Validate every fragment before mutating any state, so a bad fragment never\n // leaves the record half-replaced OR the index dimension half-established\n // (whole-record-replace must be all-or-nothing). The effective dimension is the\n // established one, or — on a still-dimensionless index — the first fragment's\n // length; it is only committed to `this._dimension` once the whole batch passes.\n const stored: IStoredFragment[] = [];\n let dimension: number | undefined = this._dimension;\n for (const fragment of fragments) {\n if (fragment.vector.length === 0) {\n return Promise.resolve(fail(`fragment index: cannot add '${key}': empty fragment vector`));\n }\n // A fragment carrying neither identity cannot be resolved back to anything by a\n // consumer holding the hit — the same invariant `embeddedFragmentConverter`\n // enforces at the untyped boundary, re-checked here at the index seam.\n if (fragment.locator === undefined && fragment.fragmentId === undefined) {\n return Promise.resolve(\n fail(\n `fragment index: cannot add '${key}': fragment requires at least one of 'locator' or 'fragmentId'`\n )\n );\n }\n if (dimension === undefined) {\n dimension = fragment.vector.length;\n } else if (fragment.vector.length !== dimension) {\n return Promise.resolve(\n fail(\n `fragment index: cannot add '${key}': fragment dimension ${fragment.vector.length} does not match index dimension ${dimension}`\n )\n );\n }\n // Defensive copy: the caller may reuse or mutate the buffer after `addFragments`.\n stored.push({ identity: fragmentIdentity(fragment), vector: Float32Array.from(fragment.vector) });\n }\n // Whole-record replace: an empty `fragments` array drops the record entirely\n // rather than leaving an empty shell behind. Commit the (possibly newly-derived)\n // dimension only alongside a successful, non-empty store.\n if (stored.length === 0) {\n this._records.delete(key);\n } else {\n this._dimension = dimension;\n this._records.set(key, { target, fragments: stored });\n }\n return Promise.resolve(succeed(stored.length));\n }\n\n /** {@inheritDoc IFragmentVectorIndex.has} */\n public has(target: IEdgeTarget): Promise<Result<boolean>> {\n return Promise.resolve(succeed(this._records.has(edgeTargetKey(target))));\n }\n\n /** {@inheritDoc IFragmentVectorIndex.remove} */\n public remove(target: IEdgeTarget): Promise<Result<IEdgeTarget>> {\n this._records.delete(edgeTargetKey(target));\n return Promise.resolve(succeed(target));\n }\n\n /** {@inheritDoc IFragmentVectorIndex.query} */\n public query(\n vector: Float32Array,\n topK: number,\n maxPerRecord?: number\n ): Promise<Result<ReadonlyArray<IVectorQueryHit>>> {\n if (topK <= 0 || this._records.size === 0) {\n return Promise.resolve(succeed([]));\n }\n if (vector.length !== this._dimension) {\n return Promise.resolve(\n fail(\n `fragment index: query dimension ${vector.length} does not match index dimension ${this._dimension}`\n )\n );\n }\n const queryMagnitude: number = InMemoryFragmentCosineIndex._magnitude(vector);\n const scored: IScoredFragment[] = [];\n for (const record of this._records.values()) {\n for (const fragment of record.fragments) {\n scored.push({\n key: edgeTargetKey(record.target),\n hit: {\n target: record.target,\n score: InMemoryFragmentCosineIndex._cosine(vector, queryMagnitude, fragment.vector),\n ...fragment.identity\n }\n });\n }\n }\n // Descending by score; the caller re-resolves each `(target, locator)` hit.\n scored.sort((a, b) => b.hit.score - a.hit.score);\n\n const hits: IVectorQueryHit[] = [];\n // Apply the per-record cap during selection (before the topK cut) so a single\n // long document cannot monopolize the result. `undefined` maxPerRecord means\n // uncapped; the counter map is always allocated (tiny) so the guard narrows\n // `maxPerRecord` directly without a non-null assertion.\n const perRecord: Map<string, number> = new Map<string, number>();\n for (const candidate of scored) {\n if (hits.length >= topK) {\n break;\n }\n if (maxPerRecord !== undefined) {\n const used: number = perRecord.get(candidate.key) ?? 0;\n if (used >= maxPerRecord) {\n continue;\n }\n perRecord.set(candidate.key, used + 1);\n }\n hits.push(candidate.hit);\n }\n return Promise.resolve(succeed(hits));\n }\n\n /**\n * Re-embed every record from `source` and rebuild the fragment index from\n * scratch. Clears the current contents (and the established dimension) first, so\n * a re-embed with a different model is supported. Returns the total number of\n * fragments indexed.\n *\n * On any failure (list, embed, or add) the index is rolled back to empty rather\n * than left in a partially-rebuilt state.\n *\n * @remarks\n * **Deliberately still returns a bare count**, unlike the record-granular\n * {@link InMemoryCosineIndex.rebuild}, which reports an\n * {@link IVectorRebuildReport}. The asymmetry is scope, not oversight: the\n * fragment path is tracked separately and gains the same treatment when the\n * `IVectorIndex`/`IFragmentVectorIndex` contracts are revisited together.\n *\n * @param source - The scope-qualified record source to re-embed.\n * @param embed - The fragment embedder applied to each record.\n */\n /** {@inheritDoc IFragmentVectorIndex.rebuild} */\n public async rebuild(\n source: IMemoryRecordSource,\n embed: FragmentEmbedder,\n options?: IVectorRebuildOptions\n ): Promise<DetailedResult<IFragmentVectorRebuildReport, IFragmentVectorRebuildReport>> {\n const lenient: boolean = (options?.onRecordError ?? 'fail') === 'skip';\n const listed: Result<IMemoryRecordListing> = await invokeHook(() => source.list());\n if (listed.isFailure()) {\n // Deliberately BEFORE the reset, matching the record-granular sibling: a\n // failed list is no evidence about the fragments already held, and nothing\n // has been re-embedded yet, so there is no half-rebuilt state to guard\n // against. Discarding a healthy index over a transient read error is data\n // loss, not caution. No detail either — an all-zero report would describe\n // an index this call never touched.\n return failWithDetail(`fragment index rebuild: failed to list records: ${listed.message}`);\n }\n // From here a rebuild is genuinely starting, so clear. A mid-loop failure\n // under `'fail'` still resets, which is what keeps that contract honest.\n this._reset();\n const indexed: Map<Kind, number> = new Map<Kind, number>();\n const fragments: Map<Kind, number> = new Map<Kind, number>();\n const declined: Map<Kind, number> = new Map<Kind, number>();\n const skipped: ISkippedVectorRecord[] = [];\n // Only the source knows what it filtered, so an absent `excluded` propagates\n // as absent rather than becoming an empty map. Before this contract existed\n // the fragment path dropped this tally on the floor, having nowhere honest to\n // put it.\n const report = (): IFragmentVectorRebuildReport => ({\n indexed,\n fragments,\n declined,\n excluded: listed.value.excluded,\n skipped\n });\n for (const scoped of listed.value.records) {\n const kind: Kind = scoped.record.envelope.kind;\n // Consumer-supplied, so a throw or rejection is captured rather than\n // escaping mid-loop and leaving the index half-populated.\n const embedded: Result<ReadonlyArray<IEmbeddedFragment>> = await invokeHook(() => embed(scoped.record));\n if (embedded.isFailure()) {\n const error: string = `fragment index rebuild: embedding '${edgeTargetKey(scoped.target)}' failed: ${\n embedded.message\n }`;\n if (!lenient) {\n this._reset();\n return failWithDetail(error, report());\n }\n skipped.push({ target: scoped.target, error });\n continue;\n }\n // An empty array is this lane's decline. It still performs a real\n // whole-record-replace — which is what clears any stale fragments — so it\n // is written, then counted as declined rather than indexed.\n const added: Result<number> = await this.addFragments(scoped.target, embedded.value);\n if (added.isFailure()) {\n const error: string = `fragment index rebuild: ${added.message}`;\n if (!lenient) {\n this._reset();\n return failWithDetail(error, report());\n }\n skipped.push({ target: scoped.target, error });\n continue;\n }\n if (added.value === 0) {\n tally(declined, kind);\n continue;\n }\n // Tallied per successful add rather than read back off the counts at the\n // end, so the per-kind buckets line up with their siblings.\n tally(indexed, kind);\n tally(fragments, kind, added.value);\n }\n return succeedWithDetail(report());\n }\n\n /** Empty the index and forget the established dimension. */\n private _reset(): void {\n this._records.clear();\n this._dimension = undefined;\n }\n\n /** The Euclidean magnitude (L2 norm) of a vector. */\n private static _magnitude(vector: Float32Array): number {\n let sum: number = 0;\n for (let i: number = 0; i < vector.length; i++) {\n sum += vector[i] * vector[i];\n }\n return Math.sqrt(sum);\n }\n\n /**\n * Cosine similarity between the query (whose magnitude is precomputed once and\n * reused across the scan) and a stored fragment vector. A zero-magnitude vector\n * on either side yields `0` rather than `NaN`.\n */\n private static _cosine(query: Float32Array, queryMagnitude: number, stored: Float32Array): number {\n const storedMagnitude: number = InMemoryFragmentCosineIndex._magnitude(stored);\n if (queryMagnitude === 0 || storedMagnitude === 0) {\n return 0;\n }\n let dot: number = 0;\n for (let i: number = 0; i < query.length; i++) {\n dot += query[i] * stored[i];\n }\n return dot / (queryMagnitude * storedMagnitude);\n }\n}\n"]}
|
|
1
|
+
{"version":3,"file":"inMemoryFragmentCosineIndex.js","sourceRoot":"","sources":["../../../src/packlets/vector/inMemoryFragmentCosineIndex.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAA0B,IAAI,EAAE,cAAc,EAAE,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AACzG,OAAO,EAA+C,aAAa,EAAE,MAAM,UAAU,CAAC;AAatF,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAUrD,0FAA0F;AAC1F,SAAS,gBAAgB,CAAC,QAA2B;IACnD,uCACK,CAAC,QAAQ,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,GACrE,CAAC,QAAQ,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EACjF;AACJ,CAAC;AAoBD;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,OAAO,2BAA2B;IAUtC;QACE,IAAI,CAAC,QAAQ,GAAG,IAAI,GAAG,EAAkC,CAAC;QAC1D,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;IAC9B,CAAC;IAED,8EAA8E;IAC9E,IAAW,WAAW;QACpB,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;IAC5B,CAAC;IAED,uEAAuE;IACvE,IAAW,aAAa;QACtB,IAAI,KAAK,GAAW,CAAC,CAAC;QACtB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;YAC5C,KAAK,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC;QACnC,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,iCAAiC;IAC1B,MAAM,CAAC,MAAM;QAClB,OAAO,OAAO,CAAC,IAAI,2BAA2B,EAAE,CAAC,CAAC;IACpD,CAAC;IAED,sDAAsD;IAC/C,YAAY,CACjB,MAAmB,EACnB,SAA2C;QAE3C,MAAM,GAAG,GAAW,aAAa,CAAC,MAAM,CAAC,CAAC;QAC1C,6EAA6E;QAC7E,0EAA0E;QAC1E,gFAAgF;QAChF,8EAA8E;QAC9E,iFAAiF;QACjF,MAAM,MAAM,GAAsB,EAAE,CAAC;QACrC,IAAI,SAAS,GAAuB,IAAI,CAAC,UAAU,CAAC;QACpD,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,IAAI,QAAQ,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACjC,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,+BAA+B,GAAG,0BAA0B,CAAC,CAAC,CAAC;YAC7F,CAAC;YACD,gFAAgF;YAChF,4EAA4E;YAC5E,uEAAuE;YACvE,IAAI,QAAQ,CAAC,OAAO,KAAK,SAAS,IAAI,QAAQ,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;gBACxE,OAAO,OAAO,CAAC,OAAO,CACpB,IAAI,CACF,+BAA+B,GAAG,gEAAgE,CACnG,CACF,CAAC;YACJ,CAAC;YACD,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;gBAC5B,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC;YACrC,CAAC;iBAAM,IAAI,QAAQ,CAAC,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;gBAChD,OAAO,OAAO,CAAC,OAAO,CACpB,IAAI,CACF,+BAA+B,GAAG,yBAAyB,QAAQ,CAAC,MAAM,CAAC,MAAM,mCAAmC,SAAS,EAAE,CAChI,CACF,CAAC;YACJ,CAAC;YACD,kFAAkF;YAClF,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,gBAAgB,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACpG,CAAC;QACD,6EAA6E;QAC7E,iFAAiF;QACjF,0DAA0D;QAC1D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC5B,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;YAC5B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;QACxD,CAAC;QACD,OAAO,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IACjD,CAAC;IAED,6CAA6C;IACtC,GAAG,CAAC,MAAmB;QAC5B,OAAO,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5E,CAAC;IAED,gDAAgD;IACzC,MAAM,CAAC,MAAmB;QAC/B,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC;QAC5C,OAAO,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;IAC1C,CAAC;IAED,+CAA+C;IACxC,KAAK,CACV,MAAoB,EACpB,IAAY,EACZ,OAA+B;;QAE/B,MAAM,YAAY,GAAuB,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,YAAY,CAAC;QAC/D,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YAC1C,OAAO,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;QACtC,CAAC;QACD,IAAI,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC;YACtC,OAAO,OAAO,CAAC,OAAO,CACpB,IAAI,CACF,mCAAmC,MAAM,CAAC,MAAM,mCAAmC,IAAI,CAAC,UAAU,EAAE,CACrG,CACF,CAAC;QACJ,CAAC;QACD,MAAM,cAAc,GAAW,2BAA2B,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAC9E,MAAM,MAAM,GAAsB,EAAE,CAAC;QACrC,6EAA6E;QAC7E,uEAAuE;QACvE,+EAA+E;QAC/E,+EAA+E;QAC/E,sBAAsB;QACtB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,EAAE,CAAC;YAClD,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;gBACxC,MAAM,CAAC,IAAI,CAAC;oBACV,GAAG,EAAE,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC;oBACjC,GAAG,kBACD,MAAM,EAAE,MAAM,CAAC,MAAM,EACrB,KAAK,EAAE,2BAA2B,CAAC,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,QAAQ,CAAC,MAAM,CAAC,IAChF,QAAQ,CAAC,QAAQ,CACrB;iBACF,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,4EAA4E;QAC5E,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAEjD,MAAM,IAAI,GAAsB,EAAE,CAAC;QACnC,8EAA8E;QAC9E,6EAA6E;QAC7E,4EAA4E;QAC5E,wDAAwD;QACxD,MAAM,SAAS,GAAwB,IAAI,GAAG,EAAkB,CAAC;QACjE,KAAK,MAAM,SAAS,IAAI,MAAM,EAAE,CAAC;YAC/B,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,EAAE,CAAC;gBACxB,MAAM;YACR,CAAC;YACD,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;gBAC/B,MAAM,IAAI,GAAW,MAAA,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,mCAAI,CAAC,CAAC;gBACvD,IAAI,IAAI,IAAI,YAAY,EAAE,CAAC;oBACzB,SAAS;gBACX,CAAC;gBACD,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC;YACzC,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAC3B,CAAC;QACD,OAAO,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;IACxC,CAAC;IAED;;;;;;;;;OASG;IACK,CAAC,cAAc,CAAC,OAA0C;QAChE,6EAA6E;QAC7E,gFAAgF;QAChF,6EAA6E;QAC7E,gBAAgB;QAChB,IAAI,OAAO,KAAK,SAAS,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YACzD,KAAK,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;YAC9B,OAAO;QACT,CAAC;QACD,MAAM,KAAK,GAAmB,OAAO,CAAC,KAAK,CAAC;QAC5C,MAAM,EAAE,GAAyB,OAAO,CAAC,EAAE,CAAC;QAC5C,IAAI,EAAE,KAAK,SAAS,EAAE,CAAC;YACrB,MAAM,MAAM,GAAuC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,aAAa,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;YACnG,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;gBACzB,MAAM,MAAM,CAAC;YACf,CAAC;YACD,OAAO;QACT,CAAC;QACD,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;YAC5C,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,KAAK,KAAK,EAAE,CAAC;gBAClC,MAAM,MAAM,CAAC;YACf,CAAC;QACH,CAAC;IACH,CAAC;IAED,iDAAiD;IAC1C,KAAK,CAAC,OAAO,CAClB,MAA2B,EAC3B,KAAuB,EACvB,OAA+B;;QAE/B,MAAM,OAAO,GAAY,CAAC,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,aAAa,mCAAI,MAAM,CAAC,KAAK,MAAM,CAAC;QACvE,MAAM,MAAM,GAAiC,MAAM,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;QACnF,IAAI,MAAM,CAAC,SAAS,EAAE,EAAE,CAAC;YACvB,yEAAyE;YACzE,2EAA2E;YAC3E,uEAAuE;YACvE,0EAA0E;YAC1E,0EAA0E;YAC1E,oCAAoC;YACpC,OAAO,cAAc,CAAC,mDAAmD,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;QAC7F,CAAC;QACD,0EAA0E;QAC1E,yEAAyE;QACzE,IAAI,CAAC,MAAM,EAAE,CAAC;QACd,MAAM,OAAO,GAAsB,IAAI,GAAG,EAAgB,CAAC;QAC3D,MAAM,SAAS,GAAsB,IAAI,GAAG,EAAgB,CAAC;QAC7D,MAAM,QAAQ,GAAsB,IAAI,GAAG,EAAgB,CAAC;QAC5D,MAAM,OAAO,GAA2B,EAAE,CAAC;QAC3C,6EAA6E;QAC7E,4EAA4E;QAC5E,8EAA8E;QAC9E,UAAU;QACV,MAAM,MAAM,GAAG,GAAiC,EAAE,CAAC,CAAC;YAClD,OAAO;YACP,SAAS;YACT,QAAQ;YACR,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ;YAC/B,OAAO;SACR,CAAC,CAAC;QACH,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;YAC1C,MAAM,IAAI,GAAS,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC;YAC/C,qEAAqE;YACrE,0DAA0D;YAC1D,MAAM,QAAQ,GAA6C,MAAM,UAAU,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;YACxG,IAAI,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC;gBACzB,MAAM,KAAK,GAAW,sCAAsC,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,aACtF,QAAQ,CAAC,OACX,EAAE,CAAC;gBACH,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,IAAI,CAAC,MAAM,EAAE,CAAC;oBACd,OAAO,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;gBACzC,CAAC;gBACD,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;gBAC/C,SAAS;YACX,CAAC;YACD,kEAAkE;YAClE,0EAA0E;YAC1E,4DAA4D;YAC5D,MAAM,KAAK,GAAmB,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;YACrF,IAAI,KAAK,CAAC,SAAS,EAAE,EAAE,CAAC;gBACtB,MAAM,KAAK,GAAW,2BAA2B,KAAK,CAAC,OAAO,EAAE,CAAC;gBACjE,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,IAAI,CAAC,MAAM,EAAE,CAAC;oBACd,OAAO,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;gBACzC,CAAC;gBACD,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;gBAC/C,SAAS;YACX,CAAC;YACD,IAAI,KAAK,CAAC,KAAK,KAAK,CAAC,EAAE,CAAC;gBACtB,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;gBACtB,SAAS;YACX,CAAC;YACD,yEAAyE;YACzE,4DAA4D;YAC5D,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;YACrB,KAAK,CAAC,SAAS,EAAE,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;QACtC,CAAC;QACD,OAAO,iBAAiB,CAAC,MAAM,EAAE,CAAC,CAAC;IACrC,CAAC;IAED,4DAA4D;IACpD,MAAM;QACZ,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;IAC9B,CAAC;IAED,qDAAqD;IAC7C,MAAM,CAAC,UAAU,CAAC,MAAoB;QAC5C,IAAI,GAAG,GAAW,CAAC,CAAC;QACpB,KAAK,IAAI,CAAC,GAAW,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/C,GAAG,IAAI,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAC/B,CAAC;QACD,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACxB,CAAC;IAED;;;;OAIG;IACK,MAAM,CAAC,OAAO,CAAC,KAAmB,EAAE,cAAsB,EAAE,MAAoB;QACtF,MAAM,eAAe,GAAW,2BAA2B,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAC/E,IAAI,cAAc,KAAK,CAAC,IAAI,eAAe,KAAK,CAAC,EAAE,CAAC;YAClD,OAAO,CAAC,CAAC;QACX,CAAC;QACD,IAAI,GAAG,GAAW,CAAC,CAAC;QACpB,KAAK,IAAI,CAAC,GAAW,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC9C,GAAG,IAAI,KAAK,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAC9B,CAAC;QACD,OAAO,GAAG,GAAG,CAAC,cAAc,GAAG,eAAe,CAAC,CAAC;IAClD,CAAC;CACF","sourcesContent":["/*\n * Copyright (c) 2026 Erik Fortune\n * SPDX-License-Identifier: MIT\n */\n\nimport { DetailedResult, Result, fail, failWithDetail, succeed, succeedWithDetail } from '@fgv/ts-utils';\nimport { IEdgeTarget, Kind, MemoryId, MemoryScopeKey, edgeTargetKey } from '../types';\nimport {\n FragmentEmbedder,\n IEmbeddedFragment,\n IFragmentQueryOptions,\n IFragmentVectorIndex,\n IFragmentVectorRebuildReport,\n IMemoryRecordListing,\n IMemoryRecordSource,\n ISkippedVectorRecord,\n IVectorQueryHit,\n IVectorRebuildOptions\n} from './vectorIndex';\nimport { invokeHook, tally } from './rebuildHelpers';\n\n/**\n * The identity fields a fragment was added with, already in query-hit shape: a field\n * the fragment did not carry is *absent*, never present-but-`undefined`, so a hit\n * for a fragment added without a `fragmentId` is structurally identical to one\n * produced before `fragmentId` existed.\n */\ntype FragmentIdentity = Pick<IVectorQueryHit, 'locator' | 'fragmentId'>;\n\n/** Project an incoming fragment's identity fields, dropping the ones it did not carry. */\nfunction fragmentIdentity(fragment: IEmbeddedFragment): FragmentIdentity {\n return {\n ...(fragment.locator !== undefined ? { locator: fragment.locator } : {}),\n ...(fragment.fragmentId !== undefined ? { fragmentId: fragment.fragmentId } : {})\n };\n}\n\n/** One stored fragment: the identity it was added with plus its vector. */\ninterface IStoredFragment {\n readonly identity: FragmentIdentity;\n readonly vector: Float32Array;\n}\n\n/** Every stored fragment for one record, tagged with the record's scoped address. */\ninterface IStoredRecordFragments {\n readonly target: IEdgeTarget;\n readonly fragments: IStoredFragment[];\n}\n\n/** A candidate hit carried through selection: the fragment's key, hit, and score. */\ninterface IScoredFragment {\n readonly key: string;\n readonly hit: IVectorQueryHit;\n}\n\n/**\n * The brute-force, in-memory cosine {@link IFragmentVectorIndex} — the\n * fragment-granular sibling of {@link InMemoryCosineIndex}. Stores many\n * `Float32Array`s per record (one per {@link IEmbeddedFragment | fragment}) and\n * answers a query by computing cosine similarity against every stored fragment,\n * returning the top-k fragment hits by descending score. Each hit carries back\n * whichever of {@link IFragmentLocator | locator} / `fragmentId` its fragment was\n * added with; a fragment must carry at least one of the two.\n *\n * @remarks\n * Same regime and same non-goals as {@link InMemoryCosineIndex}: no external\n * dependency, no ANN structure, a linear scan over the stored fragments — the seam\n * ({@link IFragmentVectorIndex}) stays open for a consumer to swap a persistent /\n * ANN backend once N grows. `addFragments` is whole-record-replace, so re-authoring\n * a document never leaves stale fragments behind. The index has a single dimension\n * established by the first fragment added; every subsequent fragment and every\n * `query` vector must match it or fail loudly — a mismatched dimension is an\n * embedder-wiring bug, never a silent zero-similarity result.\n * {@link InMemoryFragmentCosineIndex.rebuild | rebuild} clears the index (and the\n * established dimension), so a re-embed with a different model is supported.\n *\n * The optional `maxPerRecord` cap on `query` is applied **during selection**, before\n * the `topK` cut, so one long document with many strong fragments cannot crowd every\n * other record out of the result.\n * @public\n */\nexport class InMemoryFragmentCosineIndex implements IFragmentVectorIndex {\n /**\n * Stored fragments keyed by the canonical {@link edgeTargetKey} of the record's\n * scope-qualified address, so two records that share a filename stem across\n * scopes occupy distinct entries and never overwrite each other's fragments.\n */\n private readonly _records: Map<string, IStoredRecordFragments>;\n /** The dimension of every stored fragment vector; `undefined` until the first `add`. */\n private _dimension: number | undefined;\n\n private constructor() {\n this._records = new Map<string, IStoredRecordFragments>();\n this._dimension = undefined;\n }\n\n /** The number of records that currently have at least one stored fragment. */\n public get recordCount(): number {\n return this._records.size;\n }\n\n /** The total number of fragments currently held across all records. */\n public get fragmentCount(): number {\n let total: number = 0;\n for (const record of this._records.values()) {\n total += record.fragments.length;\n }\n return total;\n }\n\n /** Family-convention factory. */\n public static create(): Result<InMemoryFragmentCosineIndex> {\n return succeed(new InMemoryFragmentCosineIndex());\n }\n\n /** {@inheritDoc IFragmentVectorIndex.addFragments} */\n public addFragments(\n target: IEdgeTarget,\n fragments: ReadonlyArray<IEmbeddedFragment>\n ): Promise<Result<number>> {\n const key: string = edgeTargetKey(target);\n // Validate every fragment before mutating any state, so a bad fragment never\n // leaves the record half-replaced OR the index dimension half-established\n // (whole-record-replace must be all-or-nothing). The effective dimension is the\n // established one, or — on a still-dimensionless index — the first fragment's\n // length; it is only committed to `this._dimension` once the whole batch passes.\n const stored: IStoredFragment[] = [];\n let dimension: number | undefined = this._dimension;\n for (const fragment of fragments) {\n if (fragment.vector.length === 0) {\n return Promise.resolve(fail(`fragment index: cannot add '${key}': empty fragment vector`));\n }\n // A fragment carrying neither identity cannot be resolved back to anything by a\n // consumer holding the hit — the same invariant `embeddedFragmentConverter`\n // enforces at the untyped boundary, re-checked here at the index seam.\n if (fragment.locator === undefined && fragment.fragmentId === undefined) {\n return Promise.resolve(\n fail(\n `fragment index: cannot add '${key}': fragment requires at least one of 'locator' or 'fragmentId'`\n )\n );\n }\n if (dimension === undefined) {\n dimension = fragment.vector.length;\n } else if (fragment.vector.length !== dimension) {\n return Promise.resolve(\n fail(\n `fragment index: cannot add '${key}': fragment dimension ${fragment.vector.length} does not match index dimension ${dimension}`\n )\n );\n }\n // Defensive copy: the caller may reuse or mutate the buffer after `addFragments`.\n stored.push({ identity: fragmentIdentity(fragment), vector: Float32Array.from(fragment.vector) });\n }\n // Whole-record replace: an empty `fragments` array drops the record entirely\n // rather than leaving an empty shell behind. Commit the (possibly newly-derived)\n // dimension only alongside a successful, non-empty store.\n if (stored.length === 0) {\n this._records.delete(key);\n } else {\n this._dimension = dimension;\n this._records.set(key, { target, fragments: stored });\n }\n return Promise.resolve(succeed(stored.length));\n }\n\n /** {@inheritDoc IFragmentVectorIndex.has} */\n public has(target: IEdgeTarget): Promise<Result<boolean>> {\n return Promise.resolve(succeed(this._records.has(edgeTargetKey(target))));\n }\n\n /** {@inheritDoc IFragmentVectorIndex.remove} */\n public remove(target: IEdgeTarget): Promise<Result<IEdgeTarget>> {\n this._records.delete(edgeTargetKey(target));\n return Promise.resolve(succeed(target));\n }\n\n /** {@inheritDoc IFragmentVectorIndex.query} */\n public query(\n vector: Float32Array,\n topK: number,\n options?: IFragmentQueryOptions\n ): Promise<Result<ReadonlyArray<IVectorQueryHit>>> {\n const maxPerRecord: number | undefined = options?.maxPerRecord;\n if (topK <= 0 || this._records.size === 0) {\n return Promise.resolve(succeed([]));\n }\n if (vector.length !== this._dimension) {\n return Promise.resolve(\n fail(\n `fragment index: query dimension ${vector.length} does not match index dimension ${this._dimension}`\n )\n );\n }\n const queryMagnitude: number = InMemoryFragmentCosineIndex._magnitude(vector);\n const scored: IScoredFragment[] = [];\n // The narrowing is applied HERE — choosing which records are scored at all —\n // rather than by filtering hits afterwards. That is the whole point: a\n // post-filter would leave `topK` applied to the global set, so a scoped search\n // would silently return fewer than `topK` whenever other records outscored the\n // target's fragments.\n for (const record of this._selectRecords(options)) {\n for (const fragment of record.fragments) {\n scored.push({\n key: edgeTargetKey(record.target),\n hit: {\n target: record.target,\n score: InMemoryFragmentCosineIndex._cosine(vector, queryMagnitude, fragment.vector),\n ...fragment.identity\n }\n });\n }\n }\n // Descending by score; the caller re-resolves each `(target, locator)` hit.\n scored.sort((a, b) => b.hit.score - a.hit.score);\n\n const hits: IVectorQueryHit[] = [];\n // Apply the per-record cap during selection (before the topK cut) so a single\n // long document cannot monopolize the result. `undefined` maxPerRecord means\n // uncapped; the counter map is always allocated (tiny) so the guard narrows\n // `maxPerRecord` directly without a non-null assertion.\n const perRecord: Map<string, number> = new Map<string, number>();\n for (const candidate of scored) {\n if (hits.length >= topK) {\n break;\n }\n if (maxPerRecord !== undefined) {\n const used: number = perRecord.get(candidate.key) ?? 0;\n if (used >= maxPerRecord) {\n continue;\n }\n perRecord.set(candidate.key, used + 1);\n }\n hits.push(candidate.hit);\n }\n return Promise.resolve(succeed(hits));\n }\n\n /**\n * The records a query is allowed to score, honoring the `scope` / `id` narrowing.\n *\n * @remarks\n * The single-record case is an O(1) map lookup rather than a scan, because the\n * record map is keyed by `edgeTargetKey`. The scope-only case (a versioned kind's\n * per-entity subtree) is a filtered walk — still bounded by the vault, but it\n * scores only the entity's own fragments, which is what makes the caller's `topK`\n * meaningful.\n */\n private *_selectRecords(options: IFragmentQueryOptions | undefined): Generator<IStoredRecordFragments> {\n // Narrowed once, deliberately: re-deriving through `options?.` a second time\n // after an early return that already implies `options !== undefined` creates an\n // optional-chain arm that cannot fire, which is a dead branch rather than an\n // untested one.\n if (options === undefined || options.scope === undefined) {\n yield* this._records.values();\n return;\n }\n const scope: MemoryScopeKey = options.scope;\n const id: MemoryId | undefined = options.id;\n if (id !== undefined) {\n const record: IStoredRecordFragments | undefined = this._records.get(edgeTargetKey({ scope, id }));\n if (record !== undefined) {\n yield record;\n }\n return;\n }\n for (const record of this._records.values()) {\n if (record.target.scope === scope) {\n yield record;\n }\n }\n }\n\n /** {@inheritDoc IFragmentVectorIndex.rebuild} */\n public async rebuild(\n source: IMemoryRecordSource,\n embed: FragmentEmbedder,\n options?: IVectorRebuildOptions\n ): Promise<DetailedResult<IFragmentVectorRebuildReport, IFragmentVectorRebuildReport>> {\n const lenient: boolean = (options?.onRecordError ?? 'fail') === 'skip';\n const listed: Result<IMemoryRecordListing> = await invokeHook(() => source.list());\n if (listed.isFailure()) {\n // Deliberately BEFORE the reset, matching the record-granular sibling: a\n // failed list is no evidence about the fragments already held, and nothing\n // has been re-embedded yet, so there is no half-rebuilt state to guard\n // against. Discarding a healthy index over a transient read error is data\n // loss, not caution. No detail either — an all-zero report would describe\n // an index this call never touched.\n return failWithDetail(`fragment index rebuild: failed to list records: ${listed.message}`);\n }\n // From here a rebuild is genuinely starting, so clear. A mid-loop failure\n // under `'fail'` still resets, which is what keeps that contract honest.\n this._reset();\n const indexed: Map<Kind, number> = new Map<Kind, number>();\n const fragments: Map<Kind, number> = new Map<Kind, number>();\n const declined: Map<Kind, number> = new Map<Kind, number>();\n const skipped: ISkippedVectorRecord[] = [];\n // Only the source knows what it filtered, so an absent `excluded` propagates\n // as absent rather than becoming an empty map. Before this contract existed\n // the fragment path dropped this tally on the floor, having nowhere honest to\n // put it.\n const report = (): IFragmentVectorRebuildReport => ({\n indexed,\n fragments,\n declined,\n excluded: listed.value.excluded,\n skipped\n });\n for (const scoped of listed.value.records) {\n const kind: Kind = scoped.record.envelope.kind;\n // Consumer-supplied, so a throw or rejection is captured rather than\n // escaping mid-loop and leaving the index half-populated.\n const embedded: Result<ReadonlyArray<IEmbeddedFragment>> = await invokeHook(() => embed(scoped.record));\n if (embedded.isFailure()) {\n const error: string = `fragment index rebuild: embedding '${edgeTargetKey(scoped.target)}' failed: ${\n embedded.message\n }`;\n if (!lenient) {\n this._reset();\n return failWithDetail(error, report());\n }\n skipped.push({ target: scoped.target, error });\n continue;\n }\n // An empty array is this lane's decline. It still performs a real\n // whole-record-replace — which is what clears any stale fragments — so it\n // is written, then counted as declined rather than indexed.\n const added: Result<number> = await this.addFragments(scoped.target, embedded.value);\n if (added.isFailure()) {\n const error: string = `fragment index rebuild: ${added.message}`;\n if (!lenient) {\n this._reset();\n return failWithDetail(error, report());\n }\n skipped.push({ target: scoped.target, error });\n continue;\n }\n if (added.value === 0) {\n tally(declined, kind);\n continue;\n }\n // Tallied per successful add rather than read back off the counts at the\n // end, so the per-kind buckets line up with their siblings.\n tally(indexed, kind);\n tally(fragments, kind, added.value);\n }\n return succeedWithDetail(report());\n }\n\n /** Empty the index and forget the established dimension. */\n private _reset(): void {\n this._records.clear();\n this._dimension = undefined;\n }\n\n /** The Euclidean magnitude (L2 norm) of a vector. */\n private static _magnitude(vector: Float32Array): number {\n let sum: number = 0;\n for (let i: number = 0; i < vector.length; i++) {\n sum += vector[i] * vector[i];\n }\n return Math.sqrt(sum);\n }\n\n /**\n * Cosine similarity between the query (whose magnitude is precomputed once and\n * reused across the scan) and a stored fragment vector. A zero-magnitude vector\n * on either side yields `0` rather than `NaN`.\n */\n private static _cosine(query: Float32Array, queryMagnitude: number, stored: Float32Array): number {\n const storedMagnitude: number = InMemoryFragmentCosineIndex._magnitude(stored);\n if (queryMagnitude === 0 || storedMagnitude === 0) {\n return 0;\n }\n let dot: number = 0;\n for (let i: number = 0; i < query.length; i++) {\n dot += query[i] * stored[i];\n }\n return dot / (queryMagnitude * storedMagnitude);\n }\n}\n"]}
|