@crewhaus/tool-state 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +268 -0
- package/dist/index.d.ts +43 -0
- package/dist/index.js +1858 -0
- package/dist/lib/bm25.d.ts +117 -0
- package/dist/lib/bm25.js +245 -0
- package/dist/lib/jsonl.d.ts +47 -0
- package/dist/lib/jsonl.js +102 -0
- package/dist/lib/names.d.ts +41 -0
- package/dist/lib/names.js +144 -0
- package/dist/lib/records.d.ts +66 -0
- package/dist/lib/records.js +106 -0
- package/dist/paths.d.ts +52 -0
- package/dist/paths.js +203 -0
- package/dist/store.d.ts +128 -0
- package/dist/store.js +502 -0
- package/dist/vector.d.ts +248 -0
- package/dist/vector.js +445 -0
- package/package.json +45 -0
package/dist/vector.d.ts
ADDED
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `VectorDelete`'s seam: the vector store it deletes from, the selection it
|
|
3
|
+
* runs against that store, and the two observations it is able to make.
|
|
4
|
+
*
|
|
5
|
+
* WHY THIS IS ITS OWN MODULE. Every other tool in this package touches files
|
|
6
|
+
* under the workspace and nothing else — `index.ts` says so, and
|
|
7
|
+
* `index.test.ts` asserts `scope: "internal"` with no `ioCapability` for all
|
|
8
|
+
* twenty members of `STATE_TOOLS`. A vector store is not that: one interface
|
|
9
|
+
* covers an in-process Map, a file-backed lance index and an HTTP collection
|
|
10
|
+
* on qdrant, pinecone or weaviate. `VectorDelete` therefore declares
|
|
11
|
+
* `scope: "external"` + `ioCapability: "network"`, and is deliberately NOT a
|
|
12
|
+
* member of `STATE_TOOLS` — that list's membership is exactly what the
|
|
13
|
+
* no-boundary test checks, and a list that quietly stopped meaning what it
|
|
14
|
+
* says would be worse than no test at all. `VECTOR_TOOLS` holds it instead.
|
|
15
|
+
*
|
|
16
|
+
* WHY NOTHING HERE IMPORTS `@crewhaus/vector-store`. {@link VectorDeleteTarget}
|
|
17
|
+
* is the structural slice of that package's `VectorStore` a delete needs, and
|
|
18
|
+
* every store it builds already satisfies it — so the tool costs this package
|
|
19
|
+
* no new runtime dependency. The price is that the store must be handed in by
|
|
20
|
+
* whoever built it ({@link registerVectorTarget}, the seam
|
|
21
|
+
* `@crewhaus/tool-retrieve` already uses as `registerRetrieveConfig`): nothing
|
|
22
|
+
* here can construct a store from a backend name. There is deliberately NO
|
|
23
|
+
* ambient fallback — with nothing registered the tool refuses, because an
|
|
24
|
+
* implicit empty in-process store would let a right-to-erasure call report a
|
|
25
|
+
* clean success having deleted nothing at all.
|
|
26
|
+
*
|
|
27
|
+
* WHAT CANNOT BE KNOWN HERE, and so is never claimed. The store interface
|
|
28
|
+
* exposes neither `exists` nor `get`, so per-id truth is unobtainable: a
|
|
29
|
+
* `delete(id)` that resolves means the store ACCEPTED the request, not that
|
|
30
|
+
* the id was there or that the vector is now gone. Every name below says
|
|
31
|
+
* "attempted" or "acknowledged" for that reason, and there is no
|
|
32
|
+
* deleted/missing split anywhere — a caller erasing someone's data on legal
|
|
33
|
+
* request has to be able to read this result without being misled by it.
|
|
34
|
+
* `count()` is not a way round that either: on the HTTP backends it is
|
|
35
|
+
* eventually consistent, so a count taken immediately after a delete can
|
|
36
|
+
* equal the one before it for deletes that did land.
|
|
37
|
+
*/
|
|
38
|
+
import { CrewhausError } from "@crewhaus/errors";
|
|
39
|
+
/** Most ids one call takes. Bounds the work, and the result, per call. */
|
|
40
|
+
export declare const MAX_VECTOR_IDS = 1000;
|
|
41
|
+
/**
|
|
42
|
+
* The slice of a vector store a delete needs.
|
|
43
|
+
*
|
|
44
|
+
* Structural on purpose: `@crewhaus/vector-store`'s `VectorStore` satisfies it
|
|
45
|
+
* as it stands, and a test satisfies it with a Map — no network, no fixtures,
|
|
46
|
+
* no dependency. `backend` is the store's own self-report and may be absent;
|
|
47
|
+
* it is used for display and for one fail-closed default, never as proof of
|
|
48
|
+
* anything.
|
|
49
|
+
*/
|
|
50
|
+
export type VectorDeleteTarget = {
|
|
51
|
+
readonly backend?: string;
|
|
52
|
+
delete(id: string): Promise<void>;
|
|
53
|
+
count(): Promise<number>;
|
|
54
|
+
};
|
|
55
|
+
/** Whether `count()` on this backend reads back a write it just made. */
|
|
56
|
+
export type VectorCountConsistency = "immediate" | "eventual";
|
|
57
|
+
/**
|
|
58
|
+
* What the host registers at boot.
|
|
59
|
+
*
|
|
60
|
+
* `collection` is the name the store is bound to. It is an ASSERTION by the
|
|
61
|
+
* host — the store interface exposes no way to ask — which is precisely why
|
|
62
|
+
* {@link checkCollection} refuses rather than guesses when it is absent and a
|
|
63
|
+
* protection list is in play.
|
|
64
|
+
*/
|
|
65
|
+
export type VectorTargetRegistration = {
|
|
66
|
+
readonly store: VectorDeleteTarget;
|
|
67
|
+
readonly collection?: string;
|
|
68
|
+
/** Operator-side protection list, unioned with the caller's. */
|
|
69
|
+
readonly protectedCollections?: readonly string[];
|
|
70
|
+
readonly countConsistency?: VectorCountConsistency;
|
|
71
|
+
};
|
|
72
|
+
/** Raised when the registration itself is unusable. */
|
|
73
|
+
export declare class VectorTargetError extends CrewhausError {
|
|
74
|
+
readonly name = "VectorTargetError";
|
|
75
|
+
constructor(message: string, cause?: unknown);
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Replace the registered vector store. The host calls this at boot with the
|
|
79
|
+
* store it already built for retrieval, so a delete and a search address the
|
|
80
|
+
* same collection.
|
|
81
|
+
*
|
|
82
|
+
* The shape is checked HERE rather than at the first call: a missing method
|
|
83
|
+
* discovered during an erasure request is discovered at the worst moment. The
|
|
84
|
+
* SAME argument covers the other three fields, which is why they are checked
|
|
85
|
+
* here too rather than trusted to the type: `collection` and
|
|
86
|
+
* `protectedCollections` are the whole of the gate, and `countConsistency`
|
|
87
|
+
* decides how strong a claim the result makes. Each of them used to fail
|
|
88
|
+
* OPEN when it was not what it said it was — see the three refusals below.
|
|
89
|
+
*/
|
|
90
|
+
export declare function registerVectorTarget(registration: VectorTargetRegistration): void;
|
|
91
|
+
export declare function getVectorTarget(): VectorTargetRegistration | undefined;
|
|
92
|
+
/** Test-only — back to "nothing registered", which is the shipped default. */
|
|
93
|
+
export declare function _resetVectorTarget(): void;
|
|
94
|
+
/** The backend name for the result, bounded, or `"unknown"`. */
|
|
95
|
+
export declare function describeBackend(store: VectorDeleteTarget): string;
|
|
96
|
+
/** Is `countAfter` indicative only? An explicit registration wins over the
|
|
97
|
+
* backend name, which is only a self-report. */
|
|
98
|
+
export declare function countIsIndicative(registration: VectorTargetRegistration): boolean;
|
|
99
|
+
/** A chunk id taken apart. `@crewhaus/chunker` builds `docId:index:start`. */
|
|
100
|
+
export type ChunkId = {
|
|
101
|
+
readonly docId: string;
|
|
102
|
+
readonly index: number;
|
|
103
|
+
readonly startOffset: number;
|
|
104
|
+
};
|
|
105
|
+
/**
|
|
106
|
+
* Parse a chunk id, or `undefined` when it is not one.
|
|
107
|
+
*
|
|
108
|
+
* Read from the RIGHT. A docId is routinely a URL or a path
|
|
109
|
+
* (`https://example.com/a:0:400`, `C:\docs\x.md:3:120`), so it contains
|
|
110
|
+
* colons of its own: splitting on the first colon, or demanding exactly three
|
|
111
|
+
* colon-separated fields, mis-parses those ids. The survey sketch this tool
|
|
112
|
+
* came from specified that naive split; the two numeric fields are the LAST
|
|
113
|
+
* two, and everything before them is the docId.
|
|
114
|
+
*/
|
|
115
|
+
export declare function parseChunkId(id: string): ChunkId | undefined;
|
|
116
|
+
export type RejectedVectorId = {
|
|
117
|
+
readonly id: string;
|
|
118
|
+
readonly reason: string;
|
|
119
|
+
};
|
|
120
|
+
export type VectorSelection = {
|
|
121
|
+
/** Exactly the strings `delete()` will receive — byte for byte. */
|
|
122
|
+
readonly ids: readonly string[];
|
|
123
|
+
readonly rejected: readonly RejectedVectorId[];
|
|
124
|
+
readonly duplicatesCollapsed: number;
|
|
125
|
+
};
|
|
126
|
+
/**
|
|
127
|
+
* Turn the caller's `ids` into the selection a run acts on. The dry run and
|
|
128
|
+
* the real run call THIS — there is no second, kinder preview path, which is
|
|
129
|
+
* how a dry run ends up predicting something the real run will not do.
|
|
130
|
+
*
|
|
131
|
+
* Nothing is trimmed, case-folded or otherwise normalised. A store may hold an
|
|
132
|
+
* id with a trailing space; "delete a near-miss of what you asked for" is not
|
|
133
|
+
* a thing an erasure tool may do, so the id validated is the id deleted and
|
|
134
|
+
* the id reported.
|
|
135
|
+
*/
|
|
136
|
+
export declare function selectVectorIds(ids: readonly string[], opts: {
|
|
137
|
+
readonly requireChunkIdShape: boolean;
|
|
138
|
+
}): VectorSelection;
|
|
139
|
+
export type GateRefusalCode = "protected-collection" | "collection-unknown" | "collection-mismatch";
|
|
140
|
+
export type GateRefusal = {
|
|
141
|
+
readonly code: GateRefusalCode;
|
|
142
|
+
readonly message: string;
|
|
143
|
+
};
|
|
144
|
+
/**
|
|
145
|
+
* The one gate, run ONCE before anything is deleted.
|
|
146
|
+
*
|
|
147
|
+
* Per-id would be worse than useless: a refusal on the fourth of ten ids
|
|
148
|
+
* leaves three deleted and a dry run that predicted none of it. So this
|
|
149
|
+
* answers before the first `delete()` — before the store is touched at all —
|
|
150
|
+
* and a refusal means nothing happened.
|
|
151
|
+
*
|
|
152
|
+
* Two fail-closed rules:
|
|
153
|
+
*
|
|
154
|
+
* - An UNKNOWN collection with a protection list is a refusal, not a pass.
|
|
155
|
+
* "Could not determine" is not "no": if the host did not say which
|
|
156
|
+
* collection the store is bound to, this cannot prove it is not the
|
|
157
|
+
* protected one.
|
|
158
|
+
* - A name that matches a protected one except in case is refused as
|
|
159
|
+
* protected. Collection names are case-sensitive on the real backends, so
|
|
160
|
+
* `Chunks` and `chunks` are different collections — but a protection list
|
|
161
|
+
* that misses by a capital letter is a config slip whose cost is deleted
|
|
162
|
+
* data, while a needless refusal costs one edit.
|
|
163
|
+
*/
|
|
164
|
+
export declare function checkCollection(args: {
|
|
165
|
+
readonly collection: string | undefined;
|
|
166
|
+
readonly protectedCollections: readonly string[];
|
|
167
|
+
readonly expectCollection: string | undefined;
|
|
168
|
+
}): GateRefusal | undefined;
|
|
169
|
+
export type ToolConfigRead = {
|
|
170
|
+
readonly ok: true;
|
|
171
|
+
readonly protectedCollections: readonly string[];
|
|
172
|
+
} | {
|
|
173
|
+
readonly ok: false;
|
|
174
|
+
readonly reason: string;
|
|
175
|
+
};
|
|
176
|
+
/**
|
|
177
|
+
* The per-call `tool_config.vectorDelete` block, when a serving profile
|
|
178
|
+
* declares one (0.6.0 §4.4, the seam `@crewhaus/tool-fetch` reads the same
|
|
179
|
+
* way). Both spellings of the key are accepted so a spec block can be passed
|
|
180
|
+
* verbatim.
|
|
181
|
+
*
|
|
182
|
+
* It differs from tool-fetch's block in one way, on purpose: fetch's is an
|
|
183
|
+
* ALLOW-list and REPLACES the registered one, while this is a DENY-list and is
|
|
184
|
+
* UNIONED with the registration's and the caller's. Replacing a deny-list
|
|
185
|
+
* would let a narrower block widen what may be deleted, which is the wrong
|
|
186
|
+
* direction for a gate whose failure costs data.
|
|
187
|
+
*
|
|
188
|
+
* A malformed block is a refusal, never a silent empty list — an unreadable
|
|
189
|
+
* policy is not an absent policy.
|
|
190
|
+
*/
|
|
191
|
+
export declare function readVectorToolConfig(override: unknown): ToolConfigRead;
|
|
192
|
+
export type CountObservation = {
|
|
193
|
+
readonly count: number | null;
|
|
194
|
+
readonly error?: string;
|
|
195
|
+
};
|
|
196
|
+
/**
|
|
197
|
+
* One `count()`, as an observation that may have failed.
|
|
198
|
+
*
|
|
199
|
+
* A failed count comes back as `null` WITH a reason, never as 0: "I could not
|
|
200
|
+
* read the count" and "the collection is empty" are opposite answers, and a
|
|
201
|
+
* caller reading a fabricated 0 would conclude the erasure was complete.
|
|
202
|
+
*/
|
|
203
|
+
export declare function observeCount(store: VectorDeleteTarget): Promise<CountObservation>;
|
|
204
|
+
export type VectorDeleteError = {
|
|
205
|
+
readonly id: string;
|
|
206
|
+
readonly error: string;
|
|
207
|
+
};
|
|
208
|
+
export type VectorApplyOutcome = {
|
|
209
|
+
readonly attempted: number;
|
|
210
|
+
/** `delete()` resolved. The store accepted the request — no more than that. */
|
|
211
|
+
readonly acknowledged: number;
|
|
212
|
+
readonly errors: readonly VectorDeleteError[];
|
|
213
|
+
readonly halted: boolean;
|
|
214
|
+
};
|
|
215
|
+
/**
|
|
216
|
+
* Delete each selected id, one at a time, in the selection's order.
|
|
217
|
+
*
|
|
218
|
+
* Sequential rather than `Promise.all`: the order has to be the sorted one for
|
|
219
|
+
* a partial run to be resumable, `attempted` has to mean "these ones, up to
|
|
220
|
+
* here", and a thousand parallel requests is not a thing to do to somebody's
|
|
221
|
+
* qdrant during an erasure.
|
|
222
|
+
*
|
|
223
|
+
* An id whose `delete()` throws is recorded and the run CONTINUES. A caller
|
|
224
|
+
* erasing data on request wants the other nine ids attempted, not abandoned
|
|
225
|
+
* because the fourth timed out — and the error is reported per id, so nothing
|
|
226
|
+
* is hidden. Note what the error does NOT mean: a request that failed after
|
|
227
|
+
* reaching the backend may still have deleted the vector.
|
|
228
|
+
*/
|
|
229
|
+
export declare function applyVectorDeletes(store: VectorDeleteTarget, ids: readonly string[], signal?: AbortSignal): Promise<VectorApplyOutcome>;
|
|
230
|
+
/** A store failure as one readable, bounded line. */
|
|
231
|
+
export declare function describeStoreError(err: unknown): string;
|
|
232
|
+
/**
|
|
233
|
+
* A value the store (or the host) handed over, as one bounded line — for
|
|
234
|
+
* saying what arrived when what arrived was the wrong thing.
|
|
235
|
+
*
|
|
236
|
+
* Neither shortcut is safe alone: `JSON.stringify` returns `undefined` for
|
|
237
|
+
* `undefined`, a function or a symbol, and THROWS on a BigInt; `String()`
|
|
238
|
+
* throws on a symbol. So the type name is the fallback, and nothing here can
|
|
239
|
+
* itself fail while describing a failure.
|
|
240
|
+
*/
|
|
241
|
+
export declare function describeStoreValue(value: unknown): string;
|
|
242
|
+
/** Union of the three protection lists, de-duplicated and sorted. */
|
|
243
|
+
export declare function unionProtected(...lists: ReadonlyArray<readonly string[] | undefined>): readonly string[];
|
|
244
|
+
/** Cap a reported list, saying so rather than silently shortening it. */
|
|
245
|
+
export declare function capList<T>(items: readonly T[]): {
|
|
246
|
+
listed: readonly T[];
|
|
247
|
+
truncated: boolean;
|
|
248
|
+
};
|