@crewhaus/tool-state 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +268 -0
- package/dist/index.d.ts +43 -0
- package/dist/index.js +1858 -0
- package/dist/lib/bm25.d.ts +117 -0
- package/dist/lib/bm25.js +245 -0
- package/dist/lib/jsonl.d.ts +47 -0
- package/dist/lib/jsonl.js +102 -0
- package/dist/lib/names.d.ts +41 -0
- package/dist/lib/names.js +144 -0
- package/dist/lib/records.d.ts +66 -0
- package/dist/lib/records.js +106 -0
- package/dist/paths.d.ts +52 -0
- package/dist/paths.js +203 -0
- package/dist/store.d.ts +128 -0
- package/dist/store.js +502 -0
- package/dist/vector.d.ts +248 -0
- package/dist/vector.js +445 -0
- package/package.json +45 -0
package/README.md
ADDED
|
@@ -0,0 +1,268 @@
|
|
|
1
|
+
# @crewhaus/tool-state
|
|
2
|
+
|
|
3
|
+
Durable state for a harness, and lexical retrieval over it. A directory of
|
|
4
|
+
small files under the workspace — no database, no server, no network, nothing
|
|
5
|
+
to run alongside the harness.
|
|
6
|
+
|
|
7
|
+
A context window forgets. A crew does not get to: it has to know what it has
|
|
8
|
+
already handled, how many attempts are left, where the last run got to before
|
|
9
|
+
it was killed, and what the other agent found out an hour ago. These twenty
|
|
10
|
+
tools are that memory, and a model spends no tokens deciding any of it.
|
|
11
|
+
|
|
12
|
+
The package exports each tool and the frozen `STATE_TOOLS` list a catalog
|
|
13
|
+
registers. There is **no `all-state` category yet** — `@crewhaus/tool-categories`
|
|
14
|
+
does not know this package, so a spec names the tools it wants:
|
|
15
|
+
|
|
16
|
+
```yaml
|
|
17
|
+
tools:
|
|
18
|
+
- kvGet
|
|
19
|
+
- kvSet
|
|
20
|
+
- journalAppend
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
| Tool | What it does |
|
|
24
|
+
|---|---|
|
|
25
|
+
| `KvSet` | Store a JSON value under a key, with an optional TTL and compare-and-set |
|
|
26
|
+
| `KvGet` | Read one key back, with its version and expiry |
|
|
27
|
+
| `KvDelete` | Delete a key, optionally only at the version you read |
|
|
28
|
+
| `KvList` | List a namespace, sorted, with versions and optionally values |
|
|
29
|
+
| `CounterIncrement` | Add to a durable counter, optionally refusing to cross a limit |
|
|
30
|
+
| `CounterGet` | Read one counter, or every counter |
|
|
31
|
+
| `CheckpointSave` | Save a named JSON checkpoint as a new numbered version |
|
|
32
|
+
| `CheckpointLoad` | Load the newest version, or an earlier one by number |
|
|
33
|
+
| `CheckpointList` | What there is to resume from, with version counts and labels |
|
|
34
|
+
| `JournalAppend` | Append to an append-only JSONL stream with a monotonic sequence |
|
|
35
|
+
| `JournalRead` | Read a stream back by sequence range, kind or substring |
|
|
36
|
+
| `BlackboardPost` | Leave a note for the rest of the crew under a topic |
|
|
37
|
+
| `BlackboardRead` | Read a topic, by author, tag, sequence, or latest-per-author |
|
|
38
|
+
| `NoteWrite` | Write a durable note with a title and tags |
|
|
39
|
+
| `NoteSearch` | BM25 ranked search over the notes |
|
|
40
|
+
| `IndexBuild` | Build an inverted index over a list of workspace text files |
|
|
41
|
+
| `IndexSearch` | Query that index and get the ranked files, with optional snippets |
|
|
42
|
+
| `StateExport` | Dump the whole state directory as one JSON document |
|
|
43
|
+
| `StateImport` | Restore it, merging or replacing, with a dry run first |
|
|
44
|
+
| `DedupeMark` | Record that an external id was handled, and say if it already was |
|
|
45
|
+
|
|
46
|
+
Everything lives under `.crewhaus/state` in the workspace by default; every
|
|
47
|
+
tool takes `stateDir` to point somewhere else, and that path is contained the
|
|
48
|
+
same way (see **Containment**).
|
|
49
|
+
|
|
50
|
+
There is one more tool, and it is not one of the twenty — see
|
|
51
|
+
**[VectorDelete](#vectordelete-the-one-tool-that-leaves-the-workspace)**.
|
|
52
|
+
|
|
53
|
+
## The four properties
|
|
54
|
+
|
|
55
|
+
**Containment.** Every caller-supplied path — the state directory, and the
|
|
56
|
+
file list `IndexBuild` is given — goes through the same `resolveSafe` gate as
|
|
57
|
+
`@crewhaus/tool-fs` and `@crewhaus/tool-fsx`: nothing may resolve outside
|
|
58
|
+
`process.cwd()`, including via a symlink that lives inside the workspace and
|
|
59
|
+
points out of it. Every path *beneath* the state directory is checked the same
|
|
60
|
+
way, against its real location and not just its spelling, because the escape
|
|
61
|
+
that works in practice is not `../..` — it is an ordinary-looking `kv/cache`
|
|
62
|
+
that happens to be a symlink to `/tmp`. A dangling symlink is refused too: an
|
|
63
|
+
append would otherwise follow it and create the file it points at. Keys, ids
|
|
64
|
+
and topics never become path segments; they are percent-encoded into a single
|
|
65
|
+
filename, so a key of `../../etc/passwd` is a file called
|
|
66
|
+
`%2e.%2f..%2fetc%2fpasswd`, not a traversal.
|
|
67
|
+
|
|
68
|
+
`StateImport` in `replace` mode deletes a directory the caller names, so it
|
|
69
|
+
refuses any directory that holds anything this package did not put there —
|
|
70
|
+
and the workspace root always. `mode: "merge"` is the way to write into a
|
|
71
|
+
directory that is not ours.
|
|
72
|
+
|
|
73
|
+
**Determinism.** Same inputs against the same state, same bytes out. Listings
|
|
74
|
+
are sorted by plain code-unit comparison — never `localeCompare`, whose order
|
|
75
|
+
depends on the machine's locale. Nothing samples a random source. And **no
|
|
76
|
+
tool reads the clock**: every timestamp and every TTL is computed from a `now`
|
|
77
|
+
you supply, so a TTL is reproducible, a result can be replayed, and a stored
|
|
78
|
+
record contains no timestamp at all unless you asked for one. A `now` with no
|
|
79
|
+
offset (`2026-01-02T03:04:05`) is read as **UTC**, not in the machine's zone,
|
|
80
|
+
which ECMAScript would otherwise do — the same input has to mean the same
|
|
81
|
+
instant on a laptop and on a CI runner.
|
|
82
|
+
|
|
83
|
+
**Tolerance.** A killed process leaves half-written files. Every read here
|
|
84
|
+
reports `missing`, `corrupt` or over-size as a normal result, naming the file
|
|
85
|
+
and the reason: one bad record does not fail a listing, one truncated line
|
|
86
|
+
does not fail a journal read, and a lost sequence hint is rebuilt from the log
|
|
87
|
+
itself. Reads are bounded in memory, not merely checked beforehand, and
|
|
88
|
+
anything that is not a regular file is refused rather than read — a FIFO stats
|
|
89
|
+
as empty and would block the call for ever. Nothing throws out of `execute`
|
|
90
|
+
because the last run died badly.
|
|
91
|
+
|
|
92
|
+
**Honest safety flags.** The ten writers are `destructive: true`; the ten
|
|
93
|
+
readers are `readOnly: true` and `concurrencySafe`. Every tool in `STATE_TOOLS`
|
|
94
|
+
is `scope: "internal"` and declares no `ioCapability`, because nothing in that
|
|
95
|
+
list opens a socket or spawns a process — `packages/tool-state/src/index.test.ts`
|
|
96
|
+
asserts it for all twenty, so a future tool that reaches outward cannot slip in
|
|
97
|
+
unlabelled. None of the twenty sets `requireJustification`, because none has an
|
|
98
|
+
outward side effect: they write to one directory the operator chose, and
|
|
99
|
+
`DedupeMark` exists precisely so that the tools which *do* reach outward are
|
|
100
|
+
called once.
|
|
101
|
+
|
|
102
|
+
That test is why `VectorDelete` is not in the list. It reaches a store the host
|
|
103
|
+
registered, which may be remote, so it declares `scope: "external"` with
|
|
104
|
+
`ioCapability: "network"` and does take a justification.
|
|
105
|
+
|
|
106
|
+
## Concurrency, precisely
|
|
107
|
+
|
|
108
|
+
Two agents in one workspace are the case this package is built for.
|
|
109
|
+
|
|
110
|
+
- **Compare-and-set** (`KvSet`, `KvDelete`, `CheckpointSave`): pass the
|
|
111
|
+
version you read and the write is refused if it moved. `expectedVersion: 0`
|
|
112
|
+
means "only if nobody has claimed this yet", which makes a key a lock.
|
|
113
|
+
- **Counters** and **sequence allocation** run their read-modify-write under a
|
|
114
|
+
lock file with a 2-second deadline; a lock older than 30 seconds is treated
|
|
115
|
+
as abandoned by a dead process and broken.
|
|
116
|
+
- **Journal and blackboard appends** use `O_APPEND`, so a writer that somehow
|
|
117
|
+
bypassed the lock still cannot splice a partial line into somebody else's.
|
|
118
|
+
- **`DedupeMark`** needs no lock at all: the mark is an exclusive file create,
|
|
119
|
+
so exactly one caller can win even under a retry storm.
|
|
120
|
+
|
|
121
|
+
The limits, stated plainly: the lock is **cooperative and machine-local**. It
|
|
122
|
+
does nothing about a process that ignores it, and nothing about two machines
|
|
123
|
+
sharing a state directory over NFS, where neither `O_APPEND` atomicity nor
|
|
124
|
+
lock-file exclusion is guaranteed. Sequence numbers are monotonic but **not
|
|
125
|
+
gapless** — an append that fails after its number was allocated leaves a hole,
|
|
126
|
+
so a sequence is an ordering, not a count.
|
|
127
|
+
|
|
128
|
+
## Search, precisely
|
|
129
|
+
|
|
130
|
+
`NoteSearch` and `IndexSearch` rank with Okapi BM25:
|
|
131
|
+
|
|
132
|
+
```
|
|
133
|
+
score(D,Q) = Σ_t idf(t) · ( f(t,D)·(k1+1) ) / ( f(t,D) + k1·(1−b+b·|D|/avgdl) )
|
|
134
|
+
idf(t) = ln( 1 + (N − df(t) + 0.5) / (df(t) + 0.5) )
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
`k1` (default **1.2**) sets how fast term frequency saturates; `b` (default
|
|
138
|
+
**0.75**) how hard a long document is penalised. Both are caller-tunable and
|
|
139
|
+
both are echoed in every result, so a ranking can be reproduced later. The
|
|
140
|
+
`+1` in the logarithm is the non-negative idf variant: the classic form goes
|
|
141
|
+
negative once a term appears in more than half the corpus, which perversely
|
|
142
|
+
penalises a document for containing a query term.
|
|
143
|
+
|
|
144
|
+
**It is lexical only.** No embeddings, no synonyms, no stemming, no stop-word
|
|
145
|
+
list — a document ranks because it literally contains the query's words, so a
|
|
146
|
+
note about "automobiles" will not answer a query about "cars", and "run" will
|
|
147
|
+
not match "running". That is the trade for a retriever that needs no model,
|
|
148
|
+
no vector store and no network, and that returns the same ranking every time.
|
|
149
|
+
When meaning matters, this is the cheap first pass and a model is the
|
|
150
|
+
escalation path.
|
|
151
|
+
|
|
152
|
+
`IndexBuild` takes an explicit list of paths — pair it with `Glob` or
|
|
153
|
+
`FindFiles` rather than duplicating a directory walker here. The index is a
|
|
154
|
+
snapshot and watches nothing; `IndexSearch` compares each hit's size and mtime
|
|
155
|
+
against the index and reports `stale` rather than pretending otherwise.
|
|
156
|
+
|
|
157
|
+
## VectorDelete, the one tool that leaves the workspace
|
|
158
|
+
|
|
159
|
+
| Tool | What it does |
|
|
160
|
+
|---|---|
|
|
161
|
+
| `VectorDelete` | Delete entries from the registered vector store by id, and say exactly how little can be known about the result |
|
|
162
|
+
|
|
163
|
+
It is exported, and listed in `VECTOR_TOOLS` — **not** in `STATE_TOOLS`. The
|
|
164
|
+
twenty above promise that nothing crosses a process or network boundary and a
|
|
165
|
+
test enforces that over the list's membership; a vector store may be a qdrant,
|
|
166
|
+
pinecone or weaviate collection over HTTP, so this one is labelled
|
|
167
|
+
`scope: "external"` with `ioCapability: "network"`, takes a justification, and
|
|
168
|
+
is kept out of a list whose meaning it would quietly dilute.
|
|
169
|
+
|
|
170
|
+
**It needs a store handed to it.** This package does not depend on
|
|
171
|
+
`@crewhaus/vector-store` and cannot build a store from a backend name; the host
|
|
172
|
+
registers the one it already built, the way `@crewhaus/tool-retrieve` is wired:
|
|
173
|
+
|
|
174
|
+
```ts
|
|
175
|
+
import { registerVectorTarget } from "@crewhaus/tool-state";
|
|
176
|
+
|
|
177
|
+
registerVectorTarget({
|
|
178
|
+
store: vectorStore, // any VectorStore — structurally typed, no import
|
|
179
|
+
collection: "chunks", // needed before a protection list can mean anything
|
|
180
|
+
protectedCollections: ["audit"],
|
|
181
|
+
countConsistency: "eventual", // optional; otherwise inferred from the backend
|
|
182
|
+
});
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
With nothing registered the tool refuses. It will not fall back to an empty
|
|
186
|
+
in-process store, which would let an erasure request come back as a clean
|
|
187
|
+
success having deleted nothing.
|
|
188
|
+
|
|
189
|
+
The registration is checked there and then, not at the first call: a store
|
|
190
|
+
missing `delete`/`count`, a `collection` that is not a usable name, a
|
|
191
|
+
`protectedCollections` that is not an array of names, or a `countConsistency`
|
|
192
|
+
that is neither word, each throws `VectorTargetError` at boot. Every one of
|
|
193
|
+
them would otherwise fail *open* — a bare string protection list reads one
|
|
194
|
+
character at a time and protects nothing it names, and an unrecognised
|
|
195
|
+
`countConsistency` used to make `countAfter` sound authoritative.
|
|
196
|
+
|
|
197
|
+
**What it will not claim.** The store interface exposes no `exists` and no
|
|
198
|
+
`get`, so per-id truth is unobtainable. The result therefore reports
|
|
199
|
+
`deletesAttempted` and `deletesAcknowledged` — acknowledged meaning the store
|
|
200
|
+
accepted the request — and carries `perIdOutcome: "unavailable"`. There is no
|
|
201
|
+
deleted/missing split anywhere in it, because there is no way to earn one.
|
|
202
|
+
|
|
203
|
+
`countBefore` and `countAfter` are two observations, never a measurement of
|
|
204
|
+
what was deleted. Between settling-then-recounting and saying so, this tool
|
|
205
|
+
**says so**: `countAfterIsIndicative` is true on every backend but the
|
|
206
|
+
in-process one. Settling would mean sleeping on the system clock, which no tool
|
|
207
|
+
in this package does, and a settle loop cannot tell "the backend has not caught
|
|
208
|
+
up" from "the delete did not land" — it would convert an honest unknown into a
|
|
209
|
+
confident wrong answer. A count that could not be read at all comes back as
|
|
210
|
+
`null` with the reason beside it, never as `0`.
|
|
211
|
+
|
|
212
|
+
**The gate runs once.** `protectedCollections` — the caller's, the
|
|
213
|
+
registration's and any `tool_config` block's (both spellings of the key, also
|
|
214
|
+
unioned), because a deny-list that could be narrowed by a later source is not a
|
|
215
|
+
gate — is checked before the
|
|
216
|
+
selection, before the first count, before the first delete. A refusal means the
|
|
217
|
+
store was never touched, which is the only way `dryRun` can promise anything
|
|
218
|
+
about a real run; the dry run runs the same selection code and predicts the
|
|
219
|
+
same refusal. Two fail-closed rules: a store registered with no collection name
|
|
220
|
+
refuses while a protection list is in play (it cannot be *shown* not to be the
|
|
221
|
+
protected one), and a name differing from a protected one only in case is
|
|
222
|
+
refused rather than guessed.
|
|
223
|
+
|
|
224
|
+
**Two limits worth knowing.** The collection name is asserted by the host at
|
|
225
|
+
registration; the store offers no way to confirm it, which `expectCollection`
|
|
226
|
+
mitigates but cannot remove. And containment does not reach through the seam:
|
|
227
|
+
if a host registers a lance store pointed outside the workspace, that is the
|
|
228
|
+
host's path, resolved by the store, not by this package's gate.
|
|
229
|
+
|
|
230
|
+
## What is deliberately not here
|
|
231
|
+
|
|
232
|
+
No server, no replication, no cross-machine coordination, no queue. No
|
|
233
|
+
eviction: an expired key is hidden from reads but its file stays until
|
|
234
|
+
something deletes it, because a tool that silently removed data on a read
|
|
235
|
+
would be a bad thing to have in a crash loop. No `NoteDelete` — notes are
|
|
236
|
+
overwritten by id, and removing one is a filesystem operation. No full-text
|
|
237
|
+
search over arbitrary trees: `IndexBuild` indexes the files you name.
|
|
238
|
+
|
|
239
|
+
## Layout
|
|
240
|
+
|
|
241
|
+
`src/lib/` holds the pure functions and is where the behaviour is tested
|
|
242
|
+
(`bm25.ts`, `jsonl.ts`, `names.ts`, `records.ts`); `src/store.ts` owns the
|
|
243
|
+
filesystem — atomic writes, locks, tolerant reads — and `src/index.ts` is a
|
|
244
|
+
thin wrapper that turns those into tools. `src/paths.ts` is the containment
|
|
245
|
+
gate: it started as a copy of `@crewhaus/tool-fsx`'s, and refuses everything
|
|
246
|
+
that one refuses, plus two cases this package's own layout needs — a dangling
|
|
247
|
+
symlink, and a symlink on an interior path beneath an already-approved root
|
|
248
|
+
(`resolveWithin`). `src/vector.ts` is `VectorDelete`'s seam — the structural
|
|
249
|
+
store interface, the registration, the selection and the two observations —
|
|
250
|
+
kept in its own file so the one tool that leaves the workspace is visible as
|
|
251
|
+
such, and so moving it to a package of its own is a file move.
|
|
252
|
+
|
|
253
|
+
## On disk
|
|
254
|
+
|
|
255
|
+
```
|
|
256
|
+
.crewhaus/state/
|
|
257
|
+
kv/<namespace>/<key>.json one file per key: value, version, expiry
|
|
258
|
+
counters/<name>.json
|
|
259
|
+
checkpoints/<name>/v0000001.json numbered versions, newest last
|
|
260
|
+
journal/<stream>.jsonl append-only, plus a <stream>.jsonl.seq hint
|
|
261
|
+
blackboard/<topic>.jsonl
|
|
262
|
+
notes/<id>.json
|
|
263
|
+
indexes/<name>.json the serialised inverted index
|
|
264
|
+
dedupe/<scope>/<id>.json
|
|
265
|
+
```
|
|
266
|
+
|
|
267
|
+
It is all plain text: `cat` it, diff it, commit it, or delete the directory to
|
|
268
|
+
start again.
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import type { RegisteredTool } from "@crewhaus/tool-catalog";
|
|
2
|
+
export declare const kvSet: RegisteredTool;
|
|
3
|
+
export declare const kvGet: RegisteredTool;
|
|
4
|
+
export declare const kvDelete: RegisteredTool;
|
|
5
|
+
export declare const kvList: RegisteredTool;
|
|
6
|
+
export declare const counterIncrement: RegisteredTool;
|
|
7
|
+
export declare const counterGet: RegisteredTool;
|
|
8
|
+
export declare const checkpointSave: RegisteredTool;
|
|
9
|
+
export declare const checkpointLoad: RegisteredTool;
|
|
10
|
+
export declare const checkpointList: RegisteredTool;
|
|
11
|
+
export declare const journalAppend: RegisteredTool;
|
|
12
|
+
export declare const journalRead: RegisteredTool;
|
|
13
|
+
export declare const blackboardPost: RegisteredTool;
|
|
14
|
+
export declare const blackboardRead: RegisteredTool;
|
|
15
|
+
export declare const noteWrite: RegisteredTool;
|
|
16
|
+
export declare const noteSearch: RegisteredTool;
|
|
17
|
+
export declare const indexBuild: RegisteredTool;
|
|
18
|
+
export declare const indexSearch: RegisteredTool;
|
|
19
|
+
export declare const stateExport: RegisteredTool;
|
|
20
|
+
export declare const stateImport: RegisteredTool;
|
|
21
|
+
export declare const dedupeMark: RegisteredTool;
|
|
22
|
+
export declare const vectorDelete: RegisteredTool;
|
|
23
|
+
/**
|
|
24
|
+
* Every tool this package registers that keeps to the workspace, in the order
|
|
25
|
+
* a catalog should list them.
|
|
26
|
+
*
|
|
27
|
+
* `vectorDelete` is deliberately absent: membership of this list is what the
|
|
28
|
+
* "nothing here crosses a process or network boundary" test checks, and that
|
|
29
|
+
* tool crosses one by design. It is listed in {@link VECTOR_TOOLS} instead —
|
|
30
|
+
* see `vector.ts` for why the two are kept apart.
|
|
31
|
+
*/
|
|
32
|
+
export declare const STATE_TOOLS: ReadonlyArray<RegisteredTool>;
|
|
33
|
+
/**
|
|
34
|
+
* The tools here that reach a store outside the workspace — today, one.
|
|
35
|
+
*
|
|
36
|
+
* Kept apart from {@link STATE_TOOLS} rather than folded into it: the two
|
|
37
|
+
* lists carry different promises, and a catalog registering both is saying so
|
|
38
|
+
* explicitly instead of inheriting the wrong one.
|
|
39
|
+
*/
|
|
40
|
+
export declare const VECTOR_TOOLS: ReadonlyArray<RegisteredTool>;
|
|
41
|
+
export { ToolPermissionError } from "./paths";
|
|
42
|
+
export { DEFAULT_STATE_DIR } from "./store";
|
|
43
|
+
export { MAX_VECTOR_IDS, VectorTargetError, _resetVectorTarget, countIsIndicative, getVectorTarget, parseChunkId, registerVectorTarget, selectVectorIds, type ChunkId, type VectorCountConsistency, type VectorDeleteTarget, type VectorSelection, type VectorTargetRegistration, } from "./vector";
|