@tangleai/context 0.21.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -0
- package/LICENSE +21 -0
- package/README.md +488 -0
- package/package.json +98 -0
- package/src/archive.d.ts +47 -0
- package/src/archive.js +53 -0
- package/src/environment.d.ts +66 -0
- package/src/environment.js +715 -0
- package/src/evidence.d.ts +122 -0
- package/src/evidence.js +71 -0
- package/src/index.d.ts +9 -0
- package/src/index.js +11 -0
- package/src/ledger.d.ts +354 -0
- package/src/ledger.js +1397 -0
- package/src/recall.d.ts +74 -0
- package/src/recall.js +160 -0
- package/src/retention.d.ts +24 -0
- package/src/retention.js +91 -0
- package/src/schemas/evidence.d.ts +54 -0
- package/src/schemas/evidence.js +32 -0
- package/src/schemas/ledger.d.ts +515 -0
- package/src/schemas/ledger.js +357 -0
- package/src/schemas/patch.d.ts +129 -0
- package/src/schemas/patch.js +135 -0
- package/src/storage/memory.d.ts +86 -0
- package/src/storage/memory.js +118 -0
- package/src/storage/slot.d.ts +30 -0
- package/src/storage/slot.js +107 -0
- package/src/storage/transaction.d.ts +33 -0
- package/src/storage/transaction.js +49 -0
package/CHANGELOG.md
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# @tangleai/context
|
|
2
|
+
|
|
3
|
+
## 0.21.1
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- Update the exact Jaren foundation dependencies and source pin to the published
|
|
8
|
+
0.84.3 release after verifying its AI-free archives against source-built bytes.
|
|
9
|
+
Retain Tangle's model, context and agent ownership and align the development
|
|
10
|
+
Node pin with 24.20.0. Tangle publication remains a manual author action.
|
|
11
|
+
Allow release preparation after an already committed local release while
|
|
12
|
+
preserving its record and rejecting unprepared version edits.
|
|
13
|
+
- Updated dependencies
|
|
14
|
+
- @tangleai/models@0.21.1
|
|
15
|
+
|
|
16
|
+
## 0.21.0
|
|
17
|
+
|
|
18
|
+
### Minor Changes
|
|
19
|
+
|
|
20
|
+
- Add independent model transport, evidence-backed context, and bounded agent/program packages. Preserve the JavaScript/JSDoc APIs, strict result contracts and injected host services, with JavaScript distributions and checked declarations. Toolbox browser registration delegates to Jaren's shared WebMCP contract.
|
|
21
|
+
- Receive the reusable assistant and injected slot ledger storage, preserving
|
|
22
|
+
streaming, transcript and evidenced-memory behavior with owned cleanup.
|
|
23
|
+
|
|
24
|
+
### Patch Changes
|
|
25
|
+
|
|
26
|
+
- Updated dependencies
|
|
27
|
+
- @tangleai/models@0.21.0
|
|
28
|
+
|
|
29
|
+
The source transfer is locally qualified before its first coordinated release.
|
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Joham (jklarenbeek@gmail.com)
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,488 @@
|
|
|
1
|
+
# @tangleai/context
|
|
2
|
+
|
|
3
|
+
Evidence-backed ledgers, bounded environments, recall, retention and storage contracts.
|
|
4
|
+
|
|
5
|
+
This package keeps its JS/JSDoc implementation and deterministic tests. Inject
|
|
6
|
+
fetch, storage and compiler services at the existing seams. The public source
|
|
7
|
+
exports and emitted npm JavaScript share one implementation.
|
|
8
|
+
|
|
9
|
+
## Public entries
|
|
10
|
+
|
|
11
|
+
- `@tangleai/context`
|
|
12
|
+
- `@tangleai/context/recall`
|
|
13
|
+
- `@tangleai/context/ledger`
|
|
14
|
+
- `@tangleai/context/environment`
|
|
15
|
+
- `@tangleai/context/storage/memory`
|
|
16
|
+
- `@tangleai/context/storage/slot`
|
|
17
|
+
- `@tangleai/context/schemas/ledger`
|
|
18
|
+
- `@tangleai/context/schemas/patch`
|
|
19
|
+
- `@tangleai/context/evidence`
|
|
20
|
+
- `@tangleai/context/retention`
|
|
21
|
+
- `@tangleai/context/schemas/evidence`
|
|
22
|
+
- `@tangleai/context/archive`
|
|
23
|
+
- `@tangleai/context/package.json`
|
|
24
|
+
|
|
25
|
+
See [ownership and verification](../../docs/JAREN_AI_MIGRATION.md) for source
|
|
26
|
+
provenance, installation mode, unchanged serialized identities and qualification.
|
|
27
|
+
|
|
28
|
+
## A goal that outlives the tab
|
|
29
|
+
|
|
30
|
+
```js
|
|
31
|
+
const ledger = createLedger({ storage }); // durable storage is yours
|
|
32
|
+
await ledger.setGoal({ objective: 'Reconcile July against the bank export.' });
|
|
33
|
+
|
|
34
|
+
const agent = createAgent({
|
|
35
|
+
client, toolbox, ledger,
|
|
36
|
+
budget: { turns: 40, tokens: 250_000, ms: 15 * 60_000 }, // hard stops, not warnings
|
|
37
|
+
retrieval: { memories: { tags: ['reconcile'], limit: 5 }, skills: {} },
|
|
38
|
+
});
|
|
39
|
+
await agent.resume(); // no new instruction needed
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
The active objective and everything recorded against it are composed into the **system
|
|
43
|
+
prompt of every request** — unconditionally, because it is the thing being worked on.
|
|
44
|
+
Memories and skills are *retrieved* (`retrieval`, using the ledger's own query shape) and
|
|
45
|
+
are absent unless you ask for them. Progress is appended with evidence
|
|
46
|
+
(`{ at, note, evidence }`; bounded goals also mint a stable `id`), which is what stops a resumed session redoing finished work: a
|
|
47
|
+
new agent built over the same storage reads what has already been tried rather than being
|
|
48
|
+
told.
|
|
49
|
+
|
|
50
|
+
Composition happens **in the request, never in the transcript**. `send` still returns the
|
|
51
|
+
full, uncomposed history, so the transcript you persist and hand back next turn carries the
|
|
52
|
+
immutable base prompt and the conversation — not yesterday's rendering of the goal. A goal
|
|
53
|
+
ends by being completed, paused or cleared (`setGoalStatus`); it never ends by being
|
|
54
|
+
forgotten, and there is no timeout that silently drops it.
|
|
55
|
+
|
|
56
|
+
**Budgets are refusals.** `turns` (one turn = one model call), `tokens` and `ms` each stop
|
|
57
|
+
the run with a named `stopReason` — `budget-turns`, `budget-tokens`, `budget-ms` — and a
|
|
58
|
+
message naming what remains, the same posture as `maxToolRounds`. They bound the *run*, not
|
|
59
|
+
the turn: the counters live on the agent, `spend()` reads them back and `budget.spent` seeds
|
|
60
|
+
them, so a budget survives a reload. Token accounting uses the provider's reported `usage`
|
|
61
|
+
and falls back to deterministic character accounting (4 chars ≈ 1 token) when a provider
|
|
62
|
+
reports none — stated here because a budget that silently did not apply on the runtimes
|
|
63
|
+
that report no usage would be worse than no budget.
|
|
64
|
+
|
|
65
|
+
### Atomic storage and reported lifecycle limits
|
|
66
|
+
|
|
67
|
+
The optional storage capability is
|
|
68
|
+
`mutate(scope, current => ({ next, result }))`. A scope is a key prefix string or
|
|
69
|
+
`{ prefixes: [...], keys: [...] }`. The callback is synchronous, receives a detached
|
|
70
|
+
JSON record map, and either returns a replacement map under that scope or omits
|
|
71
|
+
`next` for a read-only decision. Exceptions publish nothing; keys cannot escape
|
|
72
|
+
the scope. Every adapter write must participate in the same serialization.
|
|
73
|
+
The memory adapter executes without yielding; the DB recipe uses one immediate
|
|
74
|
+
transaction with synchronous scoped collection operations. Ledger work stages in
|
|
75
|
+
an isolated map, then compares and atomically publishes it. Contention retries
|
|
76
|
+
against current records; embedding batches are reused during retries. Exact-key
|
|
77
|
+
scopes keep ordinary slot writes independent of corpus size. Record counters
|
|
78
|
+
survive deletion and rollback so minted memory/skill addresses are never reused.
|
|
79
|
+
|
|
80
|
+
```js
|
|
81
|
+
const ledger = createLedger({
|
|
82
|
+
storage,
|
|
83
|
+
archiveLimits: { maxItems: 24, maxBytes: 262144 },
|
|
84
|
+
goalLimits: { maxEntries: 8, maxChars: 8192, maxBytes: 16384 },
|
|
85
|
+
});
|
|
86
|
+
const report = await ledger.retentionReport();
|
|
87
|
+
const composed = await ledger.composeGoal(); // { text } or a visible refusal
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
Limits are optional, nonnegative safe integers; unknown limit names are errors.
|
|
91
|
+
Automatic retention requires atomic storage. Archive `maxItems` counts live
|
|
92
|
+
round and index slots. `maxBytes` measures the serialized archive sub-map in
|
|
93
|
+
UTF-8, including slot metadata, content, durable tombstones and the latest report.
|
|
94
|
+
Oldest unreferenced slots are evicted deterministically by timestamp and name.
|
|
95
|
+
Pinned slots, newly archived batches, addresses in retained transcript context,
|
|
96
|
+
and memory/goal references are protected. Free-text references are protected
|
|
97
|
+
conservatively by address occurrence. `putArchive` commits rounds, their index,
|
|
98
|
+
evictions and reports together. An impossible budget refuses without changing
|
|
99
|
+
storage or discarding the current transcript.
|
|
100
|
+
|
|
101
|
+
Compaction uses `putArchive(entries, { immutable: true })` to reject conflicting
|
|
102
|
+
content at an existing address, including conflicts within a batch or between
|
|
103
|
+
concurrent writers. Host-named archives remain replaceable when this option is absent.
|
|
104
|
+
|
|
105
|
+
`readSlot(name)` returns text, `undefined` for an absent address, or a typed
|
|
106
|
+
`{ status: 'evicted', name, bytes, reason }` tombstone. Recall and environment
|
|
107
|
+
reads preserve that distinction. Reports and tombstones are durable; they also
|
|
108
|
+
consume the byte budget, so an indefinitely growing audit trail eventually needs
|
|
109
|
+
host action. `clearArchives()` intentionally removes conversation rounds,
|
|
110
|
+
indexes, tombstones and the archive report. Memories, goals and snapshots remain.
|
|
111
|
+
|
|
112
|
+
Goal limits apply to the active goal: raw entry count, serialized goal bytes and
|
|
113
|
+
complete prompt characters. A deterministic lossless checkpoint stores unique
|
|
114
|
+
note/evidence pairs plus every source entry id and timestamp. Only after exact
|
|
115
|
+
coverage validates does the atomic commit replace raw entries. `checkpointReducer`
|
|
116
|
+
may supply a synchronous host/model-authored checkpoint; invented, missing,
|
|
117
|
+
duplicated or changed source evidence is rejected. This is exact evidence
|
|
118
|
+
preservation, not semantic summarization. `composeGoal()` includes the objective,
|
|
119
|
+
checkpoint and uncovered entries without excerpting; `createAgent` refuses an
|
|
120
|
+
over-budget goal before requesting a model. Increasing limits or superseding the
|
|
121
|
+
goal is explicit host action. Distinct evidence cannot be compressed indefinitely.
|
|
122
|
+
Archived goals, durable snapshots and ordinary corpus slots are separate classes;
|
|
123
|
+
these limits do not claim to cap an entire host's storage usage.
|
|
124
|
+
|
|
125
|
+
Measured tradeoffs: <!--fact:ledger.retention-->On 48 seeded rounds, an eight-round oldest policy retains 16.7% of referenced addresses; protected eviction retains 100.0%, while retaining only 4.8% of unreferenced addresses. A lossless checkpoint reduces goal context from 14,841 to 4,216 characters. Impossible protected budgets are refused.<!--/fact-->
|
|
126
|
+
|
|
127
|
+
The [retention instrument](https://github.com/jklarenbeek/jarenjs/blob/main/benchmark/README.md#ledger-retention) reports all
|
|
128
|
+
address loss, exact serialized bytes, restart correctness, retrieval and refused
|
|
129
|
+
writes. Its seeded corpus is a reproducible policy test, not a universal quota or
|
|
130
|
+
real-world language-quality claim. `ledgerFootprint(records)` accounts every
|
|
131
|
+
serialized adapter byte by class, with braces reported as overhead.
|
|
132
|
+
|
|
133
|
+
### Generic guarded documents and referential evidence
|
|
134
|
+
|
|
135
|
+
`createGuardedRefiner({ read, validateProposal, apply, validateCandidate,
|
|
136
|
+
planCommit, commit, snapshot?, restore? })` owns validation, copy/apply, candidate
|
|
137
|
+
validation, planning and commit. `prepare(document, proposal)` is synchronous and
|
|
138
|
+
returns pointered errors or a candidate/plan; `commit(proposal)` serializes the
|
|
139
|
+
whole flow. The injected committer supplies atomic persistence. Snapshot/restore
|
|
140
|
+
are paired fallback hooks; failed restoration is attempted once and retains both
|
|
141
|
+
causes. No ledger paths or universal document schema exist in this engine.
|
|
142
|
+
`createRefiner` and `createClaimRefiner` are its two production consumers.
|
|
143
|
+
|
|
144
|
+
Memory evidence accepts legacy nonempty strings unchanged, or a versioned
|
|
145
|
+
`CLAIM_EVIDENCE_SCHEMA` envelope:
|
|
146
|
+
|
|
147
|
+
```json
|
|
148
|
+
{
|
|
149
|
+
"version": 1,
|
|
150
|
+
"artifacts": [{ "id": "source", "kind": "slot", "locator": "round-1" }],
|
|
151
|
+
"evidence": [{ "id": "citation", "artifact": "source", "quote": "42 rows" }],
|
|
152
|
+
"visibleEvidence": ["citation"],
|
|
153
|
+
"claims": [{ "id": "count", "text": "There are 42 rows", "critical": true,
|
|
154
|
+
"status": "supported", "evidence": ["citation"] }]
|
|
155
|
+
}
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
`validateClaimEvidence(envelope, { artifacts? })` checks shape, unique ids within
|
|
159
|
+
each record class, artifact/evidence resolution, visible evidence membership and
|
|
160
|
+
unresolved critical claims. An optional external artifact list checks admission
|
|
161
|
+
and descriptor identity; `createClaimRefiner` requires that list. `createLedger({ artifacts })` applies the same immutable host admission list
|
|
162
|
+
to memory writes and refinement. When that option is omitted, ledger memories
|
|
163
|
+
validate self-contained envelopes. The validator never fetches locators, judges
|
|
164
|
+
source authority or decides prose entailment. Host admission establishes which
|
|
165
|
+
artifacts may be named; it does not establish the truth of a claim. Existing
|
|
166
|
+
stored string evidence needs no migration; typed consumers narrow the evidence
|
|
167
|
+
union. Envelope and checkpoint versions reject unknown versions.
|
|
168
|
+
|
|
169
|
+
### Refinement — the only way durable state changes
|
|
170
|
+
|
|
171
|
+
```js
|
|
172
|
+
import { createRefiner } from '@tangleai/agents/refine';
|
|
173
|
+
import { applyJSONPatch } from '@jarenjs/json';
|
|
174
|
+
|
|
175
|
+
const refiner = createRefiner({
|
|
176
|
+
client, ledger,
|
|
177
|
+
applyPatch: (doc, patch) => applyJSONPatch(doc, patch), // injected, never imported
|
|
178
|
+
});
|
|
179
|
+
const result = await agent.send(history);
|
|
180
|
+
await refiner.refine(result); // proposes, gates, commits — or declines
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
The model is asked what it learned, and answers with an **RFC 6902 JSON Patch** over the
|
|
184
|
+
supplemental state, generated through `createStructuredOutput`. Four stages, in order:
|
|
185
|
+
the constrained schema (three verbs, a `path` *pattern*, a cap on operations); application
|
|
186
|
+
to a **copy** through the injected patch engine; validation of every resulting record
|
|
187
|
+
against the ledger's own schemas; then a snapshot and the commit. A failure at any stage
|
|
188
|
+
returns coded, pointered errors for one bounded repair and then declines — nothing is
|
|
189
|
+
half-applied, and `rollback(result.snapshot)` restores byte-identical state after one that
|
|
190
|
+
succeeded.
|
|
191
|
+
|
|
192
|
+
Two properties are asserted rather than documented:
|
|
193
|
+
|
|
194
|
+
- **The base system prompt is not a patch target.** It is not in the document a patch
|
|
195
|
+
applies to, and no path that could reach it matches the schema's pattern. There is no
|
|
196
|
+
operation a model can write that edits its own instructions.
|
|
197
|
+
- **Every stored memory carries `evidence`**, because the ledger rejects one that does not.
|
|
198
|
+
That is the mechanism by which "evidence-backed" is enforced rather than hoped for, and
|
|
199
|
+
it is why a refinement cannot launder a hallucination into durable state.
|
|
200
|
+
|
|
201
|
+
A revised memory is stored as a new record, not an edit: a different claim, with different
|
|
202
|
+
evidence, at a different time. The empty patch is a legal answer, and the schema does not
|
|
203
|
+
demand an operation — asking a model that learned nothing to produce something is exactly
|
|
204
|
+
how an invented memory gets in.
|
|
205
|
+
|
|
206
|
+
`createRefiner({ ..., deduplicate: 'exact-evidence' })` optionally skips new
|
|
207
|
+
memories whose text and evidence match byte for byte and whose tags match as a
|
|
208
|
+
multiset. It preserves case, whitespace, independent citations, complementary
|
|
209
|
+
details and conflicts; it does not merge or delete existing records. Every
|
|
210
|
+
proposal still passes validation. A retained record must survive the same patch;
|
|
211
|
+
one scheduled for removal cannot suppress its replacement.
|
|
212
|
+
|
|
213
|
+
Results include `deduplicated`, with each skipped proposal's `path`, its
|
|
214
|
+
`retainedPath` in the proposed document, and `retainedId` when the witness was
|
|
215
|
+
already stored. A repeated batch that changes nothing creates no snapshot and
|
|
216
|
+
preserves timestamps. Calls on one refiner serialize through generation and
|
|
217
|
+
commit. On atomic storage, a refinement publishes its snapshot and every write
|
|
218
|
+
in one mutation; a changed supplemental document is refused before any patch
|
|
219
|
+
index can name a different record. Four-method adapters require host coordination
|
|
220
|
+
between refiners. Their commit failures restore once; a failed restore reports
|
|
221
|
+
both causes and the recovery snapshot. Explicit rollback is a host-authorized
|
|
222
|
+
restore of recorded state and can intentionally remove later writes.
|
|
223
|
+
|
|
224
|
+
Refinement result: <!--fact:recall.dedup-->After 12 labelled waves, opt-in exact-evidence suppression stores 21 records instead of 78; state bytes fall 73.1%. Evidence recall@10 is 1.000 versus 0.667, with all labelled conflict and complement units retained. Proposals are scripted; vectors are baai/bge-m3.<!--/fact-->
|
|
225
|
+
|
|
226
|
+
The option remains off by default. The [full policy comparison](https://github.com/jklarenbeek/jarenjs/blob/main/benchmark/README.md#labelled-recall-and-repeated-refinement)
|
|
227
|
+
includes the rejected normalization, similarity and merge controls, and clearly
|
|
228
|
+
separates scripted proposals from live embeddings.
|
|
229
|
+
|
|
230
|
+
The patch schema discriminates `oneOf` branches by operation and JSON Pointer
|
|
231
|
+
path. Memory, skill and progress values cannot be exchanged; progress is append
|
|
232
|
+
only, remove carries no value, and replace/remove require a canonical numeric
|
|
233
|
+
index. Provider decoding uses the full schema with `strict: false`. Any custom
|
|
234
|
+
validator is an additional check and cannot weaken local full-schema validation.
|
|
235
|
+
|
|
236
|
+
Provider comparison: <!--fact:ledger.decoding-->openrouter, google/gemini-3.7-flash: oneOf valid; if-then rejected-full-schema (2 calls, $0.00567000 reported cost). One trial per syntax is provider acceptance evidence, not proof of grammar enforcement. Two excluded preparatory calls cost $0.00293850 and remain recorded.<!--/fact-->
|
|
237
|
+
|
|
238
|
+
### What the cheap tier does with it (measured)
|
|
239
|
+
|
|
240
|
+
Refinement is open-ended authoring, which the field notes above say weak models do badly —
|
|
241
|
+
so it was measured on the qwen tier rather than assumed, on a four-step incident-diagnosis
|
|
242
|
+
run with facts planted in the tool results (`REC0007`, `pg-bouncer`, `v4.19.2`), so that
|
|
243
|
+
grounding and invention are both checkable without a judge.
|
|
244
|
+
|
|
245
|
+
These figures are **dated, not regenerated** — measured 2026-08-12 by a live probe that
|
|
246
|
+
needs a key and fifteen model calls, so it is not part of the committed benchmark suite and
|
|
247
|
+
not driven by the figure gate the numbers above it are. Read them as a record of one run on
|
|
248
|
+
one day, and re-run the probe rather than trusting the table if it matters.
|
|
249
|
+
|
|
250
|
+
| model | trials | accepted | first attempt | records | grounded | fabricated ids | median |
|
|
251
|
+
|---|---|---|---|---|---|---|---|
|
|
252
|
+
| `qwen/qwen3.6-35b-a3b` (non-streamed, the shipped path) | 5 | 5 | 5 | 16 | 16/16 | 0 | 69 s |
|
|
253
|
+
| `qwen/qwen3.6-35b-a3b` (streamed) | 5 | 5 | 5 | 16 | 16/16 | 0 | 56 s |
|
|
254
|
+
| `qwen/qwen3.6-27b` (streamed) | 5 | 5 | 5 | 15 | 15/15 | 0 | 10 s |
|
|
255
|
+
|
|
256
|
+
**Refinement does not need a stronger model** — with one caveat that is the whole finding.
|
|
257
|
+
Before the prompt carried a per-path shape table and one worked example, *every* trial
|
|
258
|
+
failed its first attempt and needed the repair round, always the same way: a progress entry
|
|
259
|
+
written in a memory's shape (`text` where the goal wants `note`). The schema cannot rule
|
|
260
|
+
that out — one `value` union serves three paths — so it is the prompt's job. Six of six
|
|
261
|
+
first attempts failed without it; twenty-four of twenty-four passed with it. That is this
|
|
262
|
+
package's own field note ("a few-shot example fixes *shape*") applied to its own harness,
|
|
263
|
+
and it is the difference between refinement costing one call and costing two.
|
|
264
|
+
|
|
265
|
+
Nothing else needed a stronger model: 47 of 47 records across every tier cited something
|
|
266
|
+
that was actually in the run, and no trial invented an identifier. The scoring is
|
|
267
|
+
deliberately narrow — it checks that a claim quotes the run and that no `REC…`/`v…`/region
|
|
268
|
+
token appears that the run never contained — so read it as "does not fabricate the things
|
|
269
|
+
we can check", not as a quality score.
|
|
270
|
+
|
|
271
|
+
One incidental result, recorded because the long-horizon benchmark found the opposite:
|
|
272
|
+
`createStructuredOutput` sends `stream: false`, and non-streaming did **not** hang here on
|
|
273
|
+
the same provider. It was slower on the thinking model (69 s against 56 s median) and
|
|
274
|
+
identical in outcome. The smaller `27b` answered in a tenth of that, from a fifth of the
|
|
275
|
+
completion tokens — a thinking model spends most of a refinement thinking.
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
## The environment — a corpus you work on, not one you read
|
|
279
|
+
|
|
280
|
+
Everything above makes a long context *fit*. The environment asks the other question:
|
|
281
|
+
why is the corpus in the request at all?
|
|
282
|
+
|
|
283
|
+
```js
|
|
284
|
+
import { createEnvironment } from '@tangleai/context/environment';
|
|
285
|
+
import { compileJsonQuery } from '@jarenjs/json/query';
|
|
286
|
+
|
|
287
|
+
const environment = createEnvironment({ ledger, compileQuery: compileJsonQuery });
|
|
288
|
+
await environment.put('report', await file.text()); // 10 MB is fine
|
|
289
|
+
await environment.chunk('report', { strategy: 'line', size: 4000 });
|
|
290
|
+
|
|
291
|
+
const agent = createAgent({ client, toolbox, environment }); // env_* tools registered
|
|
292
|
+
```
|
|
293
|
+
|
|
294
|
+
Content lives in named slots. The model sees a **digest** — name, kind, size, count, one
|
|
295
|
+
line of excerpt — and works by naming slots in operations:
|
|
296
|
+
|
|
297
|
+
| operation | answers with | never |
|
|
298
|
+
|---|---|---|
|
|
299
|
+
| `digest()` | every slot's metadata, capped, plus how many it did not list | content |
|
|
300
|
+
| `peek(name)` | metadata and the first characters | the slot |
|
|
301
|
+
| `chunk(name, …)` | addresses of the pieces, capped, plus how many more | the pieces |
|
|
302
|
+
| `grep(pattern, …)` | which slot matched, a window **around the hit**, and its offset | the slot |
|
|
303
|
+
| `select(name, query)` | the address of a new slot holding the result | the rows |
|
|
304
|
+
| `stat(name)` | counts, sizes, kinds — for one slot or a whole family | anything read |
|
|
305
|
+
| `read(name, { chars })` | exactly that many characters | more than asked |
|
|
306
|
+
|
|
307
|
+
**No operation returns bulk content.** Every result is capped by construction, so it is the
|
|
308
|
+
same size whether the slot holds 10 kB or 10 MB — that is asserted, not intended. `read` is
|
|
309
|
+
the single exception and it makes the caller state a budget, because a design where reading
|
|
310
|
+
is as easy as peeking is a design that ends up back in the transcript.
|
|
311
|
+
|
|
312
|
+
**The root view does not grow with the corpus.** Sweeping a corpus across three orders of
|
|
313
|
+
magnitude (10 kB → 10 MB, `test/context/environment-scale.test.js`), the root request stays
|
|
314
|
+
inside a 3 000-character band and moves by *tens* of characters between decades — the extra
|
|
315
|
+
digits in a chunk's index, and nothing else. The digest lists at most twelve slots and
|
|
316
|
+
reports how many it did not list; a cap that hid the difference would let a model conclude
|
|
317
|
+
a four-hundred-slot corpus is twelve slots long. The same sweep runs against an
|
|
318
|
+
asynchronous, out-of-process stub adapter, because a property that only held for the
|
|
319
|
+
in-memory default would be a property of the test.
|
|
320
|
+
|
|
321
|
+
Addresses are derived, never stored: a chunk is `parent#strategy:size/index`, so chunking
|
|
322
|
+
the same slot twice writes the same slots instead of a second copy. Shorter or empty
|
|
323
|
+
`ingest` and `chunk` replacements remove old indexed suffixes. Cached selections keep
|
|
324
|
+
their original result address. Environment caps must be finite safe integers; negative
|
|
325
|
+
slice bounds cannot expand a preview. `select` needs the
|
|
326
|
+
`compileQuery` seam and declines with a stated reason without it — naming `grep` as the way
|
|
327
|
+
around it — while every other operation is unaffected.
|
|
328
|
+
|
|
329
|
+
### The transcript is just another slot
|
|
330
|
+
|
|
331
|
+
```js
|
|
332
|
+
const agent = createAgent({ client, toolbox, environment, transcript: { window: 2 } });
|
|
333
|
+
```
|
|
334
|
+
|
|
335
|
+
The growing conversation is a long prompt too. With `transcript`, it is written whole to a
|
|
336
|
+
slot before every call and the request keeps a window of it plus the address of the rest —
|
|
337
|
+
so the request stops growing with the conversation, and an earlier round is reached the way
|
|
338
|
+
anything else is: `env_grep` for it, `env_read` at the offset it reports. Over forty
|
|
339
|
+
gathering rounds the request stays under 3 000 characters, and a value that a
|
|
340
|
+
6 000-character `historyBudget` run no longer carries comes back from a 600-character read
|
|
341
|
+
(`test/agents/transcript-slot.test.js`).
|
|
342
|
+
|
|
343
|
+
This is the alternative to `historyBudget` rather than a tuning of it: there is no budget to
|
|
344
|
+
exceed when the history is addressed instead of resent. `historyBudget` keeps working
|
|
345
|
+
exactly as it did — an agent with no `environment` is byte-identical to one built before
|
|
346
|
+
this existed — and which to reach for is the choice, not a migration.
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
## A durable ledger over @jarenjs/db
|
|
350
|
+
|
|
351
|
+
This injected adapter stores the complete ledger contract in one collection.
|
|
352
|
+
|
|
353
|
+
```js
|
|
354
|
+
import { openStore } from '@jarenjs/db';
|
|
355
|
+
import { nodeDriver } from '@jarenjs/db/node';
|
|
356
|
+
|
|
357
|
+
/**
|
|
358
|
+
* One collection is the whole schema a ledger needs: the storage key,
|
|
359
|
+
* the JSON value, and — when records carry embeddings — one packed
|
|
360
|
+
* vector column derived from `value.embedding`. `value` is deliberately
|
|
361
|
+
* untyped: the ledger stores objects, strings and arrays under the same
|
|
362
|
+
* contract, and only the vector member has to be declared.
|
|
363
|
+
*/
|
|
364
|
+
const ledgerModel = (dims) => ({
|
|
365
|
+
$model: '0.1',
|
|
366
|
+
collections: {
|
|
367
|
+
slots: {
|
|
368
|
+
schema: {
|
|
369
|
+
type: 'object',
|
|
370
|
+
properties: {
|
|
371
|
+
key: { type: 'string' },
|
|
372
|
+
value: { properties: { embedding: { type: 'array', items: { type: 'number' } } } },
|
|
373
|
+
},
|
|
374
|
+
required: ['key'],
|
|
375
|
+
},
|
|
376
|
+
key: '/key',
|
|
377
|
+
indexes: dims === undefined ? []
|
|
378
|
+
: [{ name: 'by_vec', path: '$.value.embedding', derive: 'vector', dims }],
|
|
379
|
+
},
|
|
380
|
+
},
|
|
381
|
+
});
|
|
382
|
+
|
|
383
|
+
/** A collection answer as a list — `execute` returns the bare item for one. */
|
|
384
|
+
const many = (result) => (Array.isArray(result) ? result : result === undefined ? [] : [result]);
|
|
385
|
+
|
|
386
|
+
/**
|
|
387
|
+
* A durable ledger storage adapter over one `@jarenjs/db` collection:
|
|
388
|
+
* the four methods, plus `rank` when a vector column is declared. It
|
|
389
|
+
* imports nothing from `@tangleai/context` — the storage contract is the
|
|
390
|
+
* whole interface between them.
|
|
391
|
+
*
|
|
392
|
+
* The prefix is INLINE in every query document rather than bound as an
|
|
393
|
+
* external, because a string operator only translates to SQL with a
|
|
394
|
+
* literal pattern; inlined, `keys()` and the ranked read both become a
|
|
395
|
+
* range scan over the key column. The ledger asks for a handful of
|
|
396
|
+
* distinct prefixes, so the documents are built once each and cached.
|
|
397
|
+
*/
|
|
398
|
+
export async function createDbStorage({ path = ':memory:', dims } = {}) {
|
|
399
|
+
const store = await openStore(ledgerModel(dims), { driver: nodeDriver(), path });
|
|
400
|
+
const slots = store.collection('slots');
|
|
401
|
+
const documents = new Map();
|
|
402
|
+
|
|
403
|
+
/** Every query document one prefix needs, built once. */
|
|
404
|
+
const forPrefix = (prefix) => {
|
|
405
|
+
let built = documents.get(prefix);
|
|
406
|
+
if (built !== undefined) return built;
|
|
407
|
+
const under = { '$starts-with': ['$r.key', prefix] };
|
|
408
|
+
const score = { $similarity: ['$r.value.embedding', '$q'] };
|
|
409
|
+
const mine = [{ $eq: ['$r.value.embeddedBy.model', '$model'] },
|
|
410
|
+
{ $eq: ['$r.value.embeddedBy.dims', '$dims'] }];
|
|
411
|
+
const counted = (where) => ({ $count: { $for: { r: '$[*]' }, $where: where, $return: '$r' } });
|
|
412
|
+
const ranked = {
|
|
413
|
+
$for: { r: '$[*]' },
|
|
414
|
+
$where: { $and: [under, ...mine] },
|
|
415
|
+
// the ledger re-scores and re-sorts what comes back, so this
|
|
416
|
+
// ordering only has to agree with its tie-break: score, then
|
|
417
|
+
// newest, then the key
|
|
418
|
+
$orderby: [{ $key: score, $dir: 'desc', $empty: 'least' },
|
|
419
|
+
{ $key: '$r.value.at', $dir: 'desc' }, '$r.key'],
|
|
420
|
+
$return: { key: '$r.key', score },
|
|
421
|
+
};
|
|
422
|
+
built = {
|
|
423
|
+
keys: { $for: { r: '$[*]' }, $where: under, $orderby: ['$r.key'], $return: '$r.key' },
|
|
424
|
+
ranked,
|
|
425
|
+
window: (limit) => ({ $subsequence: [ranked, 0, limit] }),
|
|
426
|
+
skipped: counted({ $and: [under, { $not: { $exists: '$r.value.embedding' } }] }),
|
|
427
|
+
held: counted({ $and: [under, { $exists: '$r.value.embedding' }] }),
|
|
428
|
+
ours: counted({ $and: [under, { $exists: '$r.value.embedding' }, ...mine] }),
|
|
429
|
+
names: { $distinct: { $for: { r: '$[*]' }, $where: under, $return: '$r.value.embeddedBy' } },
|
|
430
|
+
};
|
|
431
|
+
documents.set(prefix, built);
|
|
432
|
+
return built;
|
|
433
|
+
};
|
|
434
|
+
|
|
435
|
+
return {
|
|
436
|
+
mutate: async (prefix, transform) => store.transaction((tx) => {
|
|
437
|
+
const rows = tx.sync.collection('slots');
|
|
438
|
+
const prefixes = typeof prefix === 'string' ? [prefix] : prefix.prefixes ?? [];
|
|
439
|
+
const keys = [...new Set([...(prefix.keys ?? []), ...prefixes.flatMap((part) => many(rows.execute(forPrefix(part).keys)))])].sort();
|
|
440
|
+
const matches = (key) => (prefix.keys ?? []).includes(key) || prefixes.some((part) => key.startsWith(part));
|
|
441
|
+
const current = Object.fromEntries(keys.map((key) => [key, rows.get(key)?.value]));
|
|
442
|
+
for (const key of Object.keys(current)) if (current[key] === undefined) delete current[key];
|
|
443
|
+
const outcome = transform(current);
|
|
444
|
+
if (!outcome || typeof outcome.then === 'function') throw new TypeError('mutate callback must be synchronous');
|
|
445
|
+
if (outcome.next !== undefined) {
|
|
446
|
+
const next = JSON.parse(JSON.stringify(outcome.next));
|
|
447
|
+
if (Object.keys(next).some((key) => !matches(key))) throw new TypeError('mutation escaped its namespace');
|
|
448
|
+
for (const key of keys) if (!Object.hasOwn(next, key)) rows.delete(key);
|
|
449
|
+
for (const [key, value] of Object.entries(next)) rows.put({ key, value });
|
|
450
|
+
}
|
|
451
|
+
return outcome.result;
|
|
452
|
+
}, { mode: 'immediate' }),
|
|
453
|
+
get: async (key) => (await slots.get(key))?.value,
|
|
454
|
+
set: async (key, value) => { await slots.put({ key, value }); },
|
|
455
|
+
delete: async (key) => { await slots.delete(key); },
|
|
456
|
+
// sorted, because the ledger reads listings, the goal archive and a
|
|
457
|
+
// snapshot's entries in key order and its zero-padded sequences
|
|
458
|
+
// exist so that order is chronological
|
|
459
|
+
keys: async (prefix = '') => many(await slots.execute(forPrefix(prefix).keys)),
|
|
460
|
+
/**
|
|
461
|
+
* The optional fifth: rank where the records live. The window is the
|
|
462
|
+
* k-nearest plan — the vector column cuts the candidates, the engine
|
|
463
|
+
* orders them — and the two reports the ledger needs are counts,
|
|
464
|
+
* which push to SQL. Naming every identity costs a scan, so it is
|
|
465
|
+
* paid only when the counts prove a mixture, which is the one case
|
|
466
|
+
* that is about to refuse anyway.
|
|
467
|
+
*/
|
|
468
|
+
rank: async ({ prefix, vector, model, dims: width, limit }) => {
|
|
469
|
+
const docs = forPrefix(prefix);
|
|
470
|
+
const externals = { q: vector, model, dims: width };
|
|
471
|
+
const hits = many(await slots.execute(
|
|
472
|
+
limit === undefined ? docs.ranked : docs.window(limit), { externals }));
|
|
473
|
+
const skipped = await slots.execute(docs.skipped);
|
|
474
|
+
const held = await slots.execute(docs.held);
|
|
475
|
+
const ours = await slots.execute(docs.ours, { externals });
|
|
476
|
+
const identities = held === ours
|
|
477
|
+
? (ours === 0 ? [] : [{ model, dims: width }])
|
|
478
|
+
: many(await slots.execute(docs.names));
|
|
479
|
+
return { hits, skipped, identities };
|
|
480
|
+
},
|
|
481
|
+
// beyond the contract, and deliberately: the store is the host's to
|
|
482
|
+
// migrate, back up and explain, and hiding it would only mean
|
|
483
|
+
// opening a second one to do any of that
|
|
484
|
+
store,
|
|
485
|
+
close: () => store.close(),
|
|
486
|
+
};
|
|
487
|
+
}
|
|
488
|
+
```
|
package/package.json
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@tangleai/context",
|
|
3
|
+
"version": "0.21.1",
|
|
4
|
+
"description": "Evidence-backed ledgers, bounded environments, recall, retention and storage contracts.",
|
|
5
|
+
"license": "MIT",
|
|
6
|
+
"type": "module",
|
|
7
|
+
"main": "./src/index.js",
|
|
8
|
+
"types": "./src/index.d.ts",
|
|
9
|
+
"exports": {
|
|
10
|
+
".": {
|
|
11
|
+
"types": "./src/index.d.ts",
|
|
12
|
+
"import": "./src/index.js",
|
|
13
|
+
"default": "./src/index.js"
|
|
14
|
+
},
|
|
15
|
+
"./recall": {
|
|
16
|
+
"types": "./src/recall.d.ts",
|
|
17
|
+
"import": "./src/recall.js",
|
|
18
|
+
"default": "./src/recall.js"
|
|
19
|
+
},
|
|
20
|
+
"./ledger": {
|
|
21
|
+
"types": "./src/ledger.d.ts",
|
|
22
|
+
"import": "./src/ledger.js",
|
|
23
|
+
"default": "./src/ledger.js"
|
|
24
|
+
},
|
|
25
|
+
"./environment": {
|
|
26
|
+
"types": "./src/environment.d.ts",
|
|
27
|
+
"import": "./src/environment.js",
|
|
28
|
+
"default": "./src/environment.js"
|
|
29
|
+
},
|
|
30
|
+
"./storage/memory": {
|
|
31
|
+
"types": "./src/storage/memory.d.ts",
|
|
32
|
+
"import": "./src/storage/memory.js",
|
|
33
|
+
"default": "./src/storage/memory.js"
|
|
34
|
+
},
|
|
35
|
+
"./schemas/ledger": {
|
|
36
|
+
"types": "./src/schemas/ledger.d.ts",
|
|
37
|
+
"import": "./src/schemas/ledger.js",
|
|
38
|
+
"default": "./src/schemas/ledger.js"
|
|
39
|
+
},
|
|
40
|
+
"./schemas/patch": {
|
|
41
|
+
"types": "./src/schemas/patch.d.ts",
|
|
42
|
+
"import": "./src/schemas/patch.js",
|
|
43
|
+
"default": "./src/schemas/patch.js"
|
|
44
|
+
},
|
|
45
|
+
"./evidence": {
|
|
46
|
+
"types": "./src/evidence.d.ts",
|
|
47
|
+
"import": "./src/evidence.js",
|
|
48
|
+
"default": "./src/evidence.js"
|
|
49
|
+
},
|
|
50
|
+
"./retention": {
|
|
51
|
+
"types": "./src/retention.d.ts",
|
|
52
|
+
"import": "./src/retention.js",
|
|
53
|
+
"default": "./src/retention.js"
|
|
54
|
+
},
|
|
55
|
+
"./schemas/evidence": {
|
|
56
|
+
"types": "./src/schemas/evidence.d.ts",
|
|
57
|
+
"import": "./src/schemas/evidence.js",
|
|
58
|
+
"default": "./src/schemas/evidence.js"
|
|
59
|
+
},
|
|
60
|
+
"./archive": {
|
|
61
|
+
"types": "./src/archive.d.ts",
|
|
62
|
+
"import": "./src/archive.js",
|
|
63
|
+
"default": "./src/archive.js"
|
|
64
|
+
},
|
|
65
|
+
"./package.json": "./package.json",
|
|
66
|
+
"./storage/slot": {
|
|
67
|
+
"types": "./src/storage/slot.d.ts",
|
|
68
|
+
"import": "./src/storage/slot.js",
|
|
69
|
+
"default": "./src/storage/slot.js"
|
|
70
|
+
}
|
|
71
|
+
},
|
|
72
|
+
"engines": {
|
|
73
|
+
"node": ">=24"
|
|
74
|
+
},
|
|
75
|
+
"sideEffects": false,
|
|
76
|
+
"dependencies": {
|
|
77
|
+
"@jarenjs/core": "0.84.3",
|
|
78
|
+
"@jarenjs/validate": "0.84.3",
|
|
79
|
+
"@tangleai/models": "^0.21.1"
|
|
80
|
+
},
|
|
81
|
+
"private": false,
|
|
82
|
+
"files": [
|
|
83
|
+
"src/**/*.js",
|
|
84
|
+
"src/**/*.d.ts",
|
|
85
|
+
"README.md",
|
|
86
|
+
"LICENSE",
|
|
87
|
+
"CHANGELOG.md"
|
|
88
|
+
],
|
|
89
|
+
"publishConfig": {
|
|
90
|
+
"access": "public",
|
|
91
|
+
"registry": "https://registry.npmjs.org/"
|
|
92
|
+
},
|
|
93
|
+
"repository": {
|
|
94
|
+
"type": "git",
|
|
95
|
+
"url": "git+https://github.com/jklarenbeek/tangleai.git",
|
|
96
|
+
"directory": "packages/context"
|
|
97
|
+
}
|
|
98
|
+
}
|