@particle-academy/prism-graph 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +140 -2
- package/dist/chunk.d.ts +29 -0
- package/dist/chunk.js +5 -0
- package/dist/chunking/markdown.d.ts +10 -0
- package/dist/chunking/markdown.js +64 -0
- package/dist/chunking/window.d.ts +23 -0
- package/dist/chunking/window.js +128 -0
- package/dist/digest.d.ts +8 -0
- package/dist/digest.js +25 -0
- package/dist/errors.d.ts +10 -0
- package/dist/errors.js +14 -0
- package/dist/fact.d.ts +44 -0
- package/dist/fact.js +1 -0
- package/dist/graph.d.ts +106 -0
- package/dist/graph.js +266 -0
- package/dist/index-spec.d.ts +14 -0
- package/dist/index-spec.js +31 -0
- package/dist/index.d.ts +18 -0
- package/dist/index.js +16 -0
- package/dist/scope.d.ts +28 -0
- package/dist/scope.js +70 -0
- package/dist/storage/driver.d.ts +49 -0
- package/dist/storage/driver.js +100 -0
- package/dist/storage/schema.d.ts +31 -0
- package/dist/storage/schema.js +102 -0
- package/dist/storage/store.d.ts +129 -0
- package/dist/storage/store.js +391 -0
- package/dist/tools.d.ts +28 -0
- package/dist/tools.js +166 -0
- package/dist/trust.d.ts +56 -0
- package/dist/trust.js +73 -0
- package/dist/vectors.d.ts +55 -0
- package/dist/vectors.js +155 -0
- package/dist/visible.d.ts +6 -0
- package/dist/visible.js +34 -0
- package/dist/wake.d.ts +66 -0
- package/dist/wake.js +126 -0
- package/package.json +34 -4
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Particle Academy
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,141 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @particle-academy/prism-graph
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Long-term memory for AI agents, for TypeScript. This is the port of
|
|
4
|
+
[`particle-academy/prism-graph`](https://github.com/Particle-Academy/prism-graph)
|
|
5
|
+
(PHP, the reference). It is byte-compatible with the reference: chunk ids,
|
|
6
|
+
scope ids, embedder fingerprints and the fenced wake block match exactly,
|
|
7
|
+
checked against a golden fixture generated by the PHP package.
|
|
8
|
+
|
|
9
|
+
Node 22.16+, ESM, **zero runtime dependencies**. Storage is SQLite through
|
|
10
|
+
Node's built-in `node:sqlite`.
|
|
11
|
+
|
|
12
|
+
## What it guarantees
|
|
13
|
+
|
|
14
|
+
- **Scope is enforced in SQL, on every query.** Scopes are tenant, owner,
|
|
15
|
+
agent and thread. A reader sees its own level and the levels above it, never
|
|
16
|
+
sideways, and there is no unscoped read.
|
|
17
|
+
- **Every source records its trust class and writer.** Trust is `user`,
|
|
18
|
+
`document`, `agent`, `tool`, `subagent`, `consolidator` or `system`. The
|
|
19
|
+
caller binds both, never a model. A source's trust cannot change later.
|
|
20
|
+
- **Facts cite evidence, and a policy decides their status:**
|
|
21
|
+
- **accepted** when every cited chunk is user or document text;
|
|
22
|
+
- a **belief** when the evidence is only agent notes;
|
|
23
|
+
- a **proposal** otherwise.
|
|
24
|
+
|
|
25
|
+
An accepted fact can only be superseded by another accepted one, so tool
|
|
26
|
+
output cannot overwrite what the user said.
|
|
27
|
+
- **Facts are bi-temporal.** There is valid time (supersede) and recorded time
|
|
28
|
+
(retract, `knownAt`).
|
|
29
|
+
- **Hybrid search.** FTS5 full-text search folds case and accents. Optional
|
|
30
|
+
vectors are fused by reciprocal rank and partitioned per tenant.
|
|
31
|
+
- **Erasure cascades** to chunks, vectors, full-text entries, evidence and
|
|
32
|
+
orphaned facts, and returns counts.
|
|
33
|
+
- **Every write is all-or-nothing, including inside your own transaction**
|
|
34
|
+
(it runs in a savepoint). Concurrent writes on one connection are
|
|
35
|
+
serialised.
|
|
36
|
+
- **Recalled text is fenced as data.** Hidden characters are shown as
|
|
37
|
+
`<U+XXXX>`, and item text is quoted line by line, so stored text cannot
|
|
38
|
+
forge a header or close the fence.
|
|
39
|
+
|
|
40
|
+
## Use
|
|
41
|
+
|
|
42
|
+
```ts
|
|
43
|
+
import { DatabaseSync } from 'node:sqlite';
|
|
44
|
+
import { baseSchema, Graph, GraphStore, MarkdownChunker, NodeSqliteDriver, Origin, Scope, Trust } from '@particle-academy/prism-graph';
|
|
45
|
+
|
|
46
|
+
const db = new DatabaseSync('memory.db');
|
|
47
|
+
// Once, from your own migration tooling (the package never creates tables at runtime):
|
|
48
|
+
for (const sql of baseSchema()) db.exec(sql);
|
|
49
|
+
|
|
50
|
+
const graph = new Graph(new GraphStore(new NodeSqliteDriver(db)));
|
|
51
|
+
const memory = graph.scope(Scope.tenant('acme').withOwner('user-7').withAgent('journal'));
|
|
52
|
+
|
|
53
|
+
await memory.ingest('entry-42', 'journal://42', text, new MarkdownChunker(), new Origin(Trust.User, 'user-7'));
|
|
54
|
+
const hits = await memory.search('refund policy'); // verbatim chunks, with trust and source
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
`baseSchema()` is the complete schema for the default setup, and already
|
|
58
|
+
includes the full-text table and its triggers. Do not run `fullTextSchema()`
|
|
59
|
+
as well. Run `vectorSchema()` only if you store vectors. If a migration rebuilds
|
|
60
|
+
the chunks table (which drops its triggers), run `repairFullTextSchema()`.
|
|
61
|
+
|
|
62
|
+
### Give memory a connection of its own
|
|
63
|
+
|
|
64
|
+
SQLite transactions belong to the connection. Suppose another part of your
|
|
65
|
+
app writes on the same `DatabaseSync` while one of prism-graph's writes is in
|
|
66
|
+
progress. That write lands inside prism-graph's savepoint, and is rolled back
|
|
67
|
+
with it if prism-graph's write fails. So use a dedicated connection to the same
|
|
68
|
+
file (WAL mode is the usual setup), or send every write on the shared
|
|
69
|
+
connection through `driver.atomic()`.
|
|
70
|
+
|
|
71
|
+
## Listing sources and chunks
|
|
72
|
+
|
|
73
|
+
For a memory explorer, list what a scope can see, a page at a time:
|
|
74
|
+
|
|
75
|
+
```ts
|
|
76
|
+
const page = await memory.listSources(); // newest first, 50 per page
|
|
77
|
+
const next = await memory.listSources(page.nextCursor);
|
|
78
|
+
const chunks = await memory.listChunks(sourceId, null, 100);
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
Each call returns `{ contractVersion: 'g282.1', items, nextCursor }`, the same shape as the PHP package.
|
|
82
|
+
- A source item has `sourceId`, `title` (`metadata.title`, else the uri), `kind` (`metadata.kind`, else null),
|
|
83
|
+
`createdAt`, `chunkCount` and `trust`.
|
|
84
|
+
- A chunk item has `chunkId`, `sourceId`, `text`, `position` (its place in the source) and `provenance` (uri,
|
|
85
|
+
sha256, byte and line range, heading path, recipe, trust, writer, recordedAt).
|
|
86
|
+
- **Scoped like every read.** A source outside the scope lists as empty.
|
|
87
|
+
- **Cursors are opaque** and only work in the scope and listing that issued them.
|
|
88
|
+
- **Page size** defaults to 50 and is capped at 200. Pass a different cap as `new Graph(store, vectors, embedder,
|
|
89
|
+
policy, listMax)`.
|
|
90
|
+
- Retired sources and chunks are not listed.
|
|
91
|
+
|
|
92
|
+
## MCP tools on your own server
|
|
93
|
+
|
|
94
|
+
`memoryTools()` returns framework-neutral tool definitions: a name, a
|
|
95
|
+
description, a JSON Schema for the input, and an async `call()` that always
|
|
96
|
+
resolves to a JSON string and never rejects at the model. They can be
|
|
97
|
+
registered on any MCP server, with no MCP SDK dependency. The scope and the
|
|
98
|
+
writer are bound by your code, never by the model.
|
|
99
|
+
|
|
100
|
+
With [`@particle-academy/prism-mcp-server`](https://github.com/Particle-Academy/prism-mcp-server-ts)
|
|
101
|
+
(this is the setup the test suite runs over a real MCP exchange):
|
|
102
|
+
|
|
103
|
+
```ts
|
|
104
|
+
import { LoopbackHttpTransport, Server, ToolDefinition, type JsonObject } from '@particle-academy/prism-mcp-server';
|
|
105
|
+
import { memoryTools } from '@particle-academy/prism-graph';
|
|
106
|
+
|
|
107
|
+
const server = new Server({ server: 'journal', transport: new LoopbackHttpTransport() });
|
|
108
|
+
for (const tool of memoryTools(memory, 'agent:journal')) {
|
|
109
|
+
server.register(
|
|
110
|
+
new ToolDefinition(tool.name, tool.description, tool.inputSchema as JsonObject),
|
|
111
|
+
async (args) => ({ text: await tool.call(args), isError: false }),
|
|
112
|
+
);
|
|
113
|
+
}
|
|
114
|
+
await server.listen();
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
`inputSchema` is plain JSON Schema, so any MCP server that takes a tool's
|
|
118
|
+
JSON Schema as-is can register these tools without converting anything. No
|
|
119
|
+
schema library is needed: `call()` validates its own input either way.
|
|
120
|
+
|
|
121
|
+
The tools are `memory_search`, `memory_get`, `memory_facts`, `memory_timeline`
|
|
122
|
+
and `memory_note`:
|
|
123
|
+
- No tool asserts, accepts, supersedes or erases. Those are app decisions.
|
|
124
|
+
- A note is stored as the agent's own words, so it can only ever be a belief.
|
|
125
|
+
- Tool names may not contain the verbs reserved for the human: confirm,
|
|
126
|
+
approve, accept, reject, deny.
|
|
127
|
+
|
|
128
|
+
## Not yet in this port
|
|
129
|
+
|
|
130
|
+
- **PostgreSQL:** the driver interface is ready, but the adapter is not
|
|
131
|
+
shipped until it is tested here.
|
|
132
|
+
- **Harness adapters:** this port does not include the PHP reference's
|
|
133
|
+
harness adapters (WakeContext and TurnRecorder).
|
|
134
|
+
- **Entity graph:** typed entities, edges with notes and traversal are
|
|
135
|
+
planned for both languages together.
|
|
136
|
+
|
|
137
|
+
## Gates
|
|
138
|
+
|
|
139
|
+
```sh
|
|
140
|
+
npm run typecheck && npm run build && npx vitest run
|
|
141
|
+
```
|
package/dist/chunk.d.ts
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { Trust } from './trust.js';
|
|
2
|
+
/**
|
|
3
|
+
* One verbatim slice of a source: the text is exactly the source's UTF-8
|
|
4
|
+
* bytes in [byteStart, byteEnd). Nothing derived ever replaces it.
|
|
5
|
+
*/
|
|
6
|
+
export interface Chunk {
|
|
7
|
+
readonly id: string;
|
|
8
|
+
/** The scope-qualified internal source id. */
|
|
9
|
+
readonly sourceId: string;
|
|
10
|
+
readonly text: string;
|
|
11
|
+
readonly byteStart: number;
|
|
12
|
+
readonly byteEnd: number;
|
|
13
|
+
readonly lineStart: number;
|
|
14
|
+
readonly lineEnd: number;
|
|
15
|
+
readonly headingPath: readonly string[];
|
|
16
|
+
readonly recipe: string;
|
|
17
|
+
/** Set when the source no longer contains this text: it leaves search but stays readable by id. */
|
|
18
|
+
readonly retiredAt?: string | null;
|
|
19
|
+
/** Filled on read: the app's own key for the source. */
|
|
20
|
+
readonly sourceKey?: string | null;
|
|
21
|
+
/** Filled on read: the trust class the source was written with. */
|
|
22
|
+
readonly trust?: Trust | null;
|
|
23
|
+
}
|
|
24
|
+
/** Content-addressed chunk id, identical to the PHP reference's Chunk::idFor(). */
|
|
25
|
+
export declare function chunkId(recipe: string, sourceId: string, byteStart: number, byteEnd: number, text: string | Uint8Array): string;
|
|
26
|
+
export interface Chunker {
|
|
27
|
+
recipe(): string;
|
|
28
|
+
chunk(sourceId: string, text: string): Chunk[];
|
|
29
|
+
}
|
package/dist/chunk.js
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import { digest } from './digest.js';
|
|
2
|
+
/** Content-addressed chunk id, identical to the PHP reference's Chunk::idFor(). */
|
|
3
|
+
export function chunkId(recipe, sourceId, byteStart, byteEnd, text) {
|
|
4
|
+
return digest('prism-graph/chunk@1', [recipe, sourceId, byteStart, byteEnd, text]);
|
|
5
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { WindowChunker, type Section } from './window.js';
|
|
2
|
+
/**
|
|
3
|
+
* Markdown-aware: windows never cross an ATX heading (# .. ######), and each
|
|
4
|
+
* chunk carries the heading path above it. Headings inside fenced code blocks
|
|
5
|
+
* are ignored. The heading line belongs to the section it opens.
|
|
6
|
+
*/
|
|
7
|
+
export declare class MarkdownChunker extends WindowChunker {
|
|
8
|
+
protected name(): string;
|
|
9
|
+
protected sections(bytes: Buffer): Section[];
|
|
10
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { WindowChunker } from './window.js';
|
|
2
|
+
// PORT NOTE: the PHP reference matches these WITHOUT the /u flag, so its \s is
|
|
3
|
+
// ASCII whitespace only and "." matches any byte but \n. The lines are decoded
|
|
4
|
+
// as latin1 here (one char per byte) so the same patterns match the same bytes;
|
|
5
|
+
// JavaScript's own \s and "." are Unicode-wide and would not.
|
|
6
|
+
const FENCE = /^[ \t\n\x0B\f\r]{0,3}(```|~~~)/;
|
|
7
|
+
const HEADING = /^[ \t\n\x0B\f\r]{0,3}(#{1,6})[ \t]+([^\n]+?)[ \t#]*$/;
|
|
8
|
+
/**
|
|
9
|
+
* Markdown-aware: windows never cross an ATX heading (# .. ######), and each
|
|
10
|
+
* chunk carries the heading path above it. Headings inside fenced code blocks
|
|
11
|
+
* are ignored. The heading line belongs to the section it opens.
|
|
12
|
+
*/
|
|
13
|
+
export class MarkdownChunker extends WindowChunker {
|
|
14
|
+
name() {
|
|
15
|
+
return 'markdown';
|
|
16
|
+
}
|
|
17
|
+
sections(bytes) {
|
|
18
|
+
const sections = [];
|
|
19
|
+
let stack = new Map();
|
|
20
|
+
let sectionStart = 0;
|
|
21
|
+
let sectionPath = [];
|
|
22
|
+
let offset = 0;
|
|
23
|
+
let inFence = false;
|
|
24
|
+
const length = bytes.length;
|
|
25
|
+
while (offset < length) {
|
|
26
|
+
const nl = bytes.indexOf(0x0a, offset);
|
|
27
|
+
const lineEnd = nl === -1 ? length : nl + 1;
|
|
28
|
+
const line = phpRtrim(bytes.subarray(offset, lineEnd).toString('latin1'), '\r\n');
|
|
29
|
+
const heading = FENCE.test(line) ? null : HEADING.exec(line);
|
|
30
|
+
if (FENCE.test(line)) {
|
|
31
|
+
inFence = !inFence;
|
|
32
|
+
}
|
|
33
|
+
else if (!inFence && heading !== null) {
|
|
34
|
+
if (offset > sectionStart) {
|
|
35
|
+
sections.push([sectionStart, offset, sectionPath]);
|
|
36
|
+
}
|
|
37
|
+
// level => title; a heading closes every deeper level, even across skipped levels.
|
|
38
|
+
const level = heading[1].length;
|
|
39
|
+
stack = new Map([...stack].filter(([l]) => l < level));
|
|
40
|
+
stack.set(level, Buffer.from(phpTrim(heading[2]), 'latin1').toString('utf8'));
|
|
41
|
+
stack = new Map([...stack].sort(([a], [b]) => a - b));
|
|
42
|
+
sectionStart = offset;
|
|
43
|
+
sectionPath = [...stack.values()];
|
|
44
|
+
}
|
|
45
|
+
offset = lineEnd;
|
|
46
|
+
}
|
|
47
|
+
if (length > sectionStart) {
|
|
48
|
+
sections.push([sectionStart, length, sectionPath]);
|
|
49
|
+
}
|
|
50
|
+
return sections;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
/** PHP's trim() default: " \t\n\r\0\x0B" only. */
|
|
54
|
+
function phpTrim(value) {
|
|
55
|
+
return value.replace(/^[ \t\n\r\0\x0B]+|[ \t\n\r\0\x0B]+$/g, '');
|
|
56
|
+
}
|
|
57
|
+
/** PHP's rtrim($value, $chars) for a set of single characters. */
|
|
58
|
+
function phpRtrim(value, chars) {
|
|
59
|
+
let end = value.length;
|
|
60
|
+
while (end > 0 && chars.includes(value[end - 1])) {
|
|
61
|
+
end--;
|
|
62
|
+
}
|
|
63
|
+
return value.slice(0, end);
|
|
64
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { type Chunk, type Chunker } from '../chunk.js';
|
|
2
|
+
/** A section windows may not cross: [byteStart, byteEnd, headingPath]. */
|
|
3
|
+
export type Section = [number, number, string[]];
|
|
4
|
+
/**
|
|
5
|
+
* Byte windows with overlap, preferring paragraph, then sentence, then line
|
|
6
|
+
* breaks, and never splitting a UTF-8 character.
|
|
7
|
+
*
|
|
8
|
+
* PORT NOTE: this works on the UTF-8 BYTES, exactly as the PHP reference works
|
|
9
|
+
* on PHP strings, so spans, ids and boundaries are identical across languages
|
|
10
|
+
* (checked against a golden fixture generated by the reference).
|
|
11
|
+
*/
|
|
12
|
+
export declare class WindowChunker implements Chunker {
|
|
13
|
+
protected readonly target: number;
|
|
14
|
+
protected readonly max: number;
|
|
15
|
+
protected readonly overlap: number;
|
|
16
|
+
constructor(target?: number, max?: number, overlap?: number);
|
|
17
|
+
recipe(): string;
|
|
18
|
+
protected name(): string;
|
|
19
|
+
chunk(sourceId: string, text: string): Chunk[];
|
|
20
|
+
protected sections(bytes: Buffer): Section[];
|
|
21
|
+
private windows;
|
|
22
|
+
private breakAt;
|
|
23
|
+
}
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import { chunkId } from '../chunk.js';
|
|
2
|
+
import { InvalidArgumentError } from '../errors.js';
|
|
3
|
+
const MARKS = ['\n\n', '. ', '! ', '? ', '\n', ' '].map((m) => Buffer.from(m, 'latin1'));
|
|
4
|
+
/**
|
|
5
|
+
* Byte windows with overlap, preferring paragraph, then sentence, then line
|
|
6
|
+
* breaks, and never splitting a UTF-8 character.
|
|
7
|
+
*
|
|
8
|
+
* PORT NOTE: this works on the UTF-8 BYTES, exactly as the PHP reference works
|
|
9
|
+
* on PHP strings, so spans, ids and boundaries are identical across languages
|
|
10
|
+
* (checked against a golden fixture generated by the reference).
|
|
11
|
+
*/
|
|
12
|
+
export class WindowChunker {
|
|
13
|
+
target;
|
|
14
|
+
max;
|
|
15
|
+
overlap;
|
|
16
|
+
constructor(target = 1200, max = 1500, overlap = 150) {
|
|
17
|
+
this.target = target;
|
|
18
|
+
this.max = max;
|
|
19
|
+
this.overlap = overlap;
|
|
20
|
+
// max >= 4: one UTF-8 character can be 4 bytes.
|
|
21
|
+
if (!Number.isInteger(target) || !Number.isInteger(max) || !Number.isInteger(overlap)
|
|
22
|
+
|| target < 1 || max < target || max < 4 || overlap < 0 || overlap >= target) {
|
|
23
|
+
throw new InvalidArgumentError('Chunk sizes need 0 <= overlap < target <= max, and max >= 4.');
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
recipe() {
|
|
27
|
+
return `${this.name()}@1;target=${this.target};max=${this.max};overlap=${this.overlap}`;
|
|
28
|
+
}
|
|
29
|
+
name() {
|
|
30
|
+
return 'window';
|
|
31
|
+
}
|
|
32
|
+
chunk(sourceId, text) {
|
|
33
|
+
const bytes = Buffer.from(text, 'utf8');
|
|
34
|
+
if (bytes.length === 0) {
|
|
35
|
+
return [];
|
|
36
|
+
}
|
|
37
|
+
const lines = lineStarts(bytes);
|
|
38
|
+
const recipe = this.recipe();
|
|
39
|
+
const chunks = [];
|
|
40
|
+
for (const [start, end, path] of this.sections(bytes)) {
|
|
41
|
+
for (const [s, e] of this.windows(bytes, start, end)) {
|
|
42
|
+
const slice = bytes.subarray(s, e);
|
|
43
|
+
chunks.push({
|
|
44
|
+
id: chunkId(recipe, sourceId, s, e, slice),
|
|
45
|
+
sourceId,
|
|
46
|
+
text: slice.toString('utf8'),
|
|
47
|
+
byteStart: s,
|
|
48
|
+
byteEnd: e,
|
|
49
|
+
lineStart: lineAt(lines, s),
|
|
50
|
+
lineEnd: lineAt(lines, e - 1),
|
|
51
|
+
headingPath: path,
|
|
52
|
+
recipe,
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
return chunks;
|
|
57
|
+
}
|
|
58
|
+
sections(bytes) {
|
|
59
|
+
return [[0, bytes.length, []]];
|
|
60
|
+
}
|
|
61
|
+
windows(bytes, start, end) {
|
|
62
|
+
const windows = [];
|
|
63
|
+
let pos = start;
|
|
64
|
+
while (pos < end) {
|
|
65
|
+
const stop = end - pos <= this.max ? end : this.breakAt(bytes, pos, end);
|
|
66
|
+
windows.push([pos, stop]);
|
|
67
|
+
if (stop >= end) {
|
|
68
|
+
break;
|
|
69
|
+
}
|
|
70
|
+
let next = charBoundary(bytes, Math.max(pos + 1, stop - this.overlap));
|
|
71
|
+
// Start the overlap on a word boundary, never inside a word.
|
|
72
|
+
const space = bytes.indexOf(0x20, next);
|
|
73
|
+
if (this.overlap > 0 && space !== -1 && space < stop) {
|
|
74
|
+
next = space + 1;
|
|
75
|
+
}
|
|
76
|
+
pos = next > pos ? next : stop;
|
|
77
|
+
}
|
|
78
|
+
return windows;
|
|
79
|
+
}
|
|
80
|
+
breakAt(bytes, pos, end) {
|
|
81
|
+
const limit = Math.min(pos + this.max, end);
|
|
82
|
+
const floor = pos + Math.floor(this.target / 2);
|
|
83
|
+
const window = bytes.subarray(pos, limit);
|
|
84
|
+
for (const mark of MARKS) {
|
|
85
|
+
const at = window.lastIndexOf(mark);
|
|
86
|
+
if (at !== -1 && pos + at + mark.length > floor) {
|
|
87
|
+
return pos + at + mark.length;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
// Hard cut, on a character boundary.
|
|
91
|
+
return Math.max(pos + 1, charBoundary(bytes, limit));
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
/** Move back to the start of a UTF-8 character. */
|
|
95
|
+
function charBoundary(bytes, at) {
|
|
96
|
+
while (at > 0 && at < bytes.length && (bytes[at] & 0xc0) === 0x80) {
|
|
97
|
+
at--;
|
|
98
|
+
}
|
|
99
|
+
return at;
|
|
100
|
+
}
|
|
101
|
+
function lineStarts(bytes) {
|
|
102
|
+
const starts = [0];
|
|
103
|
+
let offset = 0;
|
|
104
|
+
for (;;) {
|
|
105
|
+
const nl = bytes.indexOf(0x0a, offset);
|
|
106
|
+
if (nl === -1) {
|
|
107
|
+
break;
|
|
108
|
+
}
|
|
109
|
+
starts.push(nl + 1);
|
|
110
|
+
offset = nl + 1;
|
|
111
|
+
}
|
|
112
|
+
return starts;
|
|
113
|
+
}
|
|
114
|
+
/** 1-based line of a byte offset. */
|
|
115
|
+
function lineAt(starts, byte) {
|
|
116
|
+
let lo = 0;
|
|
117
|
+
let hi = starts.length - 1;
|
|
118
|
+
while (lo < hi) {
|
|
119
|
+
const mid = Math.floor((lo + hi + 1) / 2);
|
|
120
|
+
if (starts[mid] <= byte) {
|
|
121
|
+
lo = mid;
|
|
122
|
+
}
|
|
123
|
+
else {
|
|
124
|
+
hi = mid - 1;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
return lo + 1;
|
|
128
|
+
}
|
package/dist/digest.d.ts
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* sha256 of a length-prefixed tuple, byte-identical to the PHP reference's
|
|
3
|
+
* Support\Digest::of(). Lengths are UTF-8 BYTE lengths, as PHP's strlen().
|
|
4
|
+
* A separator byte would let two different tuples hash the same; a length
|
|
5
|
+
* prefix cannot be forged that way.
|
|
6
|
+
*/
|
|
7
|
+
export declare function digest(domain: string, fields: ReadonlyArray<string | number | Uint8Array>): string;
|
|
8
|
+
export declare function sha256(data: string | Uint8Array): string;
|
package/dist/digest.js
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
/**
|
|
3
|
+
* sha256 of a length-prefixed tuple, byte-identical to the PHP reference's
|
|
4
|
+
* Support\Digest::of(). Lengths are UTF-8 BYTE lengths, as PHP's strlen().
|
|
5
|
+
* A separator byte would let two different tuples hash the same; a length
|
|
6
|
+
* prefix cannot be forged that way.
|
|
7
|
+
*/
|
|
8
|
+
export function digest(domain, fields) {
|
|
9
|
+
const hash = createHash('sha256');
|
|
10
|
+
const part = (field) => {
|
|
11
|
+
const bytes = typeof field === 'string' ? Buffer.from(field, 'utf8')
|
|
12
|
+
: typeof field === 'number' ? Buffer.from(String(field), 'utf8')
|
|
13
|
+
: Buffer.from(field);
|
|
14
|
+
hash.update(`${bytes.length}:`);
|
|
15
|
+
hash.update(bytes);
|
|
16
|
+
};
|
|
17
|
+
part(domain);
|
|
18
|
+
for (const field of fields) {
|
|
19
|
+
part(field);
|
|
20
|
+
}
|
|
21
|
+
return hash.digest('hex');
|
|
22
|
+
}
|
|
23
|
+
export function sha256(data) {
|
|
24
|
+
return createHash('sha256').update(data).digest('hex');
|
|
25
|
+
}
|
package/dist/errors.d.ts
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/** A refusal by the graph itself: forged ids, policy violations, pinned embedders, storage refusals. */
|
|
2
|
+
export declare class GraphError extends Error {
|
|
3
|
+
constructor(message: string, options?: {
|
|
4
|
+
cause?: unknown;
|
|
5
|
+
});
|
|
6
|
+
}
|
|
7
|
+
/** A malformed argument: a programming error in the caller, kept apart from GraphError on purpose. */
|
|
8
|
+
export declare class InvalidArgumentError extends Error {
|
|
9
|
+
constructor(message: string);
|
|
10
|
+
}
|
package/dist/errors.js
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/** A refusal by the graph itself: forged ids, policy violations, pinned embedders, storage refusals. */
|
|
2
|
+
export class GraphError extends Error {
|
|
3
|
+
constructor(message, options) {
|
|
4
|
+
super(message, options);
|
|
5
|
+
this.name = 'GraphError';
|
|
6
|
+
}
|
|
7
|
+
}
|
|
8
|
+
/** A malformed argument: a programming error in the caller, kept apart from GraphError on purpose. */
|
|
9
|
+
export class InvalidArgumentError extends Error {
|
|
10
|
+
constructor(message) {
|
|
11
|
+
super(message);
|
|
12
|
+
this.name = 'InvalidArgumentError';
|
|
13
|
+
}
|
|
14
|
+
}
|
package/dist/fact.d.ts
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import type { FactStatus } from './trust.js';
|
|
2
|
+
/**
|
|
3
|
+
* A subject-predicate-object statement, bi-temporal:
|
|
4
|
+
* - VALID time [validFrom, validTo): when it held in the world;
|
|
5
|
+
* - RECORDED time [recordedAt, expiredAt): when the memory held it.
|
|
6
|
+
*
|
|
7
|
+
* The app supplies id, subject, predicate, object, evidence and valid times.
|
|
8
|
+
* The store fills status, recordedAt, expiredAt and supersededBy; a writer
|
|
9
|
+
* that sets any of them is refused.
|
|
10
|
+
*/
|
|
11
|
+
export interface Fact {
|
|
12
|
+
readonly id: string;
|
|
13
|
+
readonly subject: string;
|
|
14
|
+
readonly predicate: string;
|
|
15
|
+
readonly object: string;
|
|
16
|
+
/** Chunk ids this fact rests on. */
|
|
17
|
+
readonly evidence: readonly string[];
|
|
18
|
+
readonly validFrom: string;
|
|
19
|
+
readonly validTo?: string | null;
|
|
20
|
+
readonly supersededBy?: string | null;
|
|
21
|
+
readonly status?: FactStatus | null;
|
|
22
|
+
readonly recordedAt?: string | null;
|
|
23
|
+
readonly expiredAt?: string | null;
|
|
24
|
+
}
|
|
25
|
+
/** What an erasure removed, as counts that can be shown as evidence. */
|
|
26
|
+
export interface EraseReport {
|
|
27
|
+
readonly sourceFound: boolean;
|
|
28
|
+
readonly chunks: number;
|
|
29
|
+
readonly evidenceLinks: number;
|
|
30
|
+
readonly factsErased: number;
|
|
31
|
+
readonly factsKept: number;
|
|
32
|
+
readonly vectors: number;
|
|
33
|
+
/** The erased chunk ids (vectors are removed by these). */
|
|
34
|
+
readonly chunkIds: readonly string[];
|
|
35
|
+
}
|
|
36
|
+
export interface IngestReport {
|
|
37
|
+
readonly source: string;
|
|
38
|
+
readonly sha256: string;
|
|
39
|
+
readonly recipe: string;
|
|
40
|
+
readonly chunks: number;
|
|
41
|
+
readonly stored: number;
|
|
42
|
+
/** Chunks of this source retired after the ingest: text it no longer contains. */
|
|
43
|
+
readonly retired: number;
|
|
44
|
+
}
|
package/dist/fact.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
package/dist/graph.d.ts
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import type { Chunk, Chunker } from './chunk.js';
|
|
2
|
+
import type { EraseReport, Fact, IngestReport } from './fact.js';
|
|
3
|
+
import type { Scope } from './scope.js';
|
|
4
|
+
import type { GraphStore, ListedChunk, ListedSource, ListingPage } from './storage/store.js';
|
|
5
|
+
import { type AcceptancePolicy, type Origin } from './trust.js';
|
|
6
|
+
import type { Embedder, VectorStore } from './vectors.js';
|
|
7
|
+
export interface Hit {
|
|
8
|
+
readonly chunk: Chunk;
|
|
9
|
+
readonly score: number;
|
|
10
|
+
/** "text", the vector store's method, or "hybrid" when both found it. */
|
|
11
|
+
readonly method: string;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Long-term memory for agents. This object only holds the wiring (store,
|
|
15
|
+
* optional vectors and embedder, acceptance policy); it has NO read or write
|
|
16
|
+
* methods of its own. Everything goes through a scope, so an unscoped query
|
|
17
|
+
* cannot be written, by an app or by a model.
|
|
18
|
+
*/
|
|
19
|
+
export declare class Graph {
|
|
20
|
+
private readonly store;
|
|
21
|
+
private readonly vectors;
|
|
22
|
+
private readonly embedder;
|
|
23
|
+
private readonly policy;
|
|
24
|
+
/** The largest page listSources/listChunks will return (G-282). */
|
|
25
|
+
private readonly listMax;
|
|
26
|
+
constructor(store: GraphStore, vectors?: VectorStore | null, embedder?: Embedder | null, policy?: AcceptancePolicy,
|
|
27
|
+
/** The largest page listSources/listChunks will return (G-282). */
|
|
28
|
+
listMax?: number);
|
|
29
|
+
scope(scope: Scope): ScopedGraph;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Memory as seen from one scope. Reads return what this scope can see (its own
|
|
33
|
+
* rows and those shared from wider scopes above it); writes land at exactly
|
|
34
|
+
* this scope. Nothing here widens a write.
|
|
35
|
+
*/
|
|
36
|
+
export declare class ScopedGraph {
|
|
37
|
+
private readonly memoryScope;
|
|
38
|
+
private readonly store;
|
|
39
|
+
private readonly vectors;
|
|
40
|
+
private readonly embedder;
|
|
41
|
+
private readonly policy;
|
|
42
|
+
private readonly listMax;
|
|
43
|
+
static readonly MAX_LIMIT = 200;
|
|
44
|
+
/** A query goes to the embedding provider and the full-text index, so it is bounded. */
|
|
45
|
+
static readonly MAX_QUERY_BYTES = 4096;
|
|
46
|
+
static readonly MAX_GET_IDS = 1000;
|
|
47
|
+
private static readonly RRF_K;
|
|
48
|
+
/** The listing contract (G-282). A consumer refuses a version it does not know. */
|
|
49
|
+
static readonly CONTRACT_VERSION: "g282.1";
|
|
50
|
+
static readonly DEFAULT_LIST_LIMIT = 50;
|
|
51
|
+
static readonly DEFAULT_LIST_MAX = 200;
|
|
52
|
+
constructor(memoryScope: Scope, store: GraphStore, vectors: VectorStore | null, embedder: Embedder | null, policy: AcceptancePolicy, listMax?: number);
|
|
53
|
+
/** Sources this scope can see, newest first, a page at a time (G-282). */
|
|
54
|
+
listSources(cursor?: string | null, limit?: number | null): Promise<ListingPage<ListedSource>>;
|
|
55
|
+
/**
|
|
56
|
+
* Live chunks of one source, in source order, a page at a time (G-282). A source outside this scope yields an
|
|
57
|
+
* empty page, never an error that would confirm it exists.
|
|
58
|
+
*/
|
|
59
|
+
listChunks(sourceId: string, cursor?: string | null, limit?: number | null): Promise<ListingPage<ListedChunk>>;
|
|
60
|
+
private listLimit;
|
|
61
|
+
/**
|
|
62
|
+
* Opaque, and bound to this scope and listing: a cursor from another scope or listing is refused. It needs no
|
|
63
|
+
* secret, because every listing query applies the scope predicate regardless of the cursor.
|
|
64
|
+
*/
|
|
65
|
+
private writeCursor;
|
|
66
|
+
private readCursor;
|
|
67
|
+
private cursorBinding;
|
|
68
|
+
scope(): Scope;
|
|
69
|
+
/**
|
|
70
|
+
* Store a source and its verbatim chunks at this scope. Idempotent per key:
|
|
71
|
+
* unchanged text stores and embeds nothing new; text the source no longer
|
|
72
|
+
* contains is retired from search. Verbatim is stored first and never waits
|
|
73
|
+
* on embedding; a failed embedding heals on the next ingest.
|
|
74
|
+
*/
|
|
75
|
+
ingest(key: string, uri: string, text: string, chunker: Chunker, origin: Origin, options?: {
|
|
76
|
+
metadata?: Record<string, unknown>;
|
|
77
|
+
sha256?: string;
|
|
78
|
+
occurredAt?: string;
|
|
79
|
+
}): Promise<IngestReport>;
|
|
80
|
+
/** Take a source at this scope out of search; it stays readable by id. */
|
|
81
|
+
retireSource(key: string): Promise<number>;
|
|
82
|
+
/** Erase a source stored at this scope: chunks, vectors, evidence links and orphaned facts. */
|
|
83
|
+
erase(key: string): Promise<EraseReport>;
|
|
84
|
+
get(chunkIds: readonly string[]): Promise<Chunk[]>;
|
|
85
|
+
/**
|
|
86
|
+
* Hybrid search over what this scope can see: full-text (BM25) and, when
|
|
87
|
+
* configured and `vectors` is true, vectors, fused by reciprocal rank. A
|
|
88
|
+
* chunk with no vector yet still ranks through text.
|
|
89
|
+
*/
|
|
90
|
+
search(query: string, limit?: number, vectors?: boolean): Promise<Hit[]>;
|
|
91
|
+
facts(subject: string, options?: {
|
|
92
|
+
at?: string | null;
|
|
93
|
+
knownAt?: string | null;
|
|
94
|
+
unaccepted?: boolean;
|
|
95
|
+
}): Promise<Fact[]>;
|
|
96
|
+
timeline(subject: string): Promise<Fact[]>;
|
|
97
|
+
/** Store a fact at this scope; its status comes from the acceptance policy. */
|
|
98
|
+
assert(fact: Fact): Promise<Fact>;
|
|
99
|
+
supersede(oldKey: string, replacement: Fact): Promise<Fact>;
|
|
100
|
+
/** An app decision, never a tool. */
|
|
101
|
+
retract(key: string): Promise<void>;
|
|
102
|
+
/** An app decision, never a tool: a proposed fact becomes accepted. */
|
|
103
|
+
accept(key: string): Promise<void>;
|
|
104
|
+
/** The vector index for this tenant: another tenant's vectors can neither crowd nor appear in its results. */
|
|
105
|
+
private spec;
|
|
106
|
+
}
|