@ixo/oracle-runtime-workers 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +110 -0
- package/package.json +77 -0
- package/src/attachments/classify.ts +114 -0
- package/src/attachments/content-blocks.ts +51 -0
- package/src/attachments/download.ts +235 -0
- package/src/attachments/extract.ts +241 -0
- package/src/attachments/index.ts +84 -0
- package/src/attachments/magic.ts +133 -0
- package/src/attachments/pipeline.ts +409 -0
- package/src/attachments/retention.ts +205 -0
- package/src/attachments/route.ts +41 -0
- package/src/attachments/sandbox-archive.ts +90 -0
- package/src/attachments/sanitize.ts +35 -0
- package/src/attachments/types.ts +86 -0
- package/src/attachments/view.ts +107 -0
- package/src/core/env.ts +200 -0
- package/src/core/index.ts +513 -0
- package/src/core/llm.ts +842 -0
- package/src/core/main-agent-types.ts +123 -0
- package/src/core/main-agent.ts +399 -0
- package/src/core/manifest.ts +365 -0
- package/src/core/meta-tools.ts +232 -0
- package/src/core/middlewares/byo-history-sanitizer.ts +165 -0
- package/src/core/middlewares/capability-gate.ts +76 -0
- package/src/core/middlewares/index.ts +18 -0
- package/src/core/middlewares/page-context.ts +98 -0
- package/src/core/middlewares/safety-guardrail.ts +117 -0
- package/src/core/middlewares/summarization.ts +99 -0
- package/src/core/middlewares/tool-repetition-guard.ts +125 -0
- package/src/core/middlewares/tool-validation.ts +83 -0
- package/src/core/middlewares/work-status.ts +57 -0
- package/src/core/openrouter-pricing.ts +124 -0
- package/src/core/plugin-loader.ts +192 -0
- package/src/core/plugins/index.ts +6 -0
- package/src/core/plugins/skills/index.ts +8 -0
- package/src/core/plugins/skills/skills-tools.ts +356 -0
- package/src/core/plugins/skills/skills.plugin.ts +113 -0
- package/src/core/plugins/weather/index.ts +8 -0
- package/src/core/plugins/weather/weather-client.ts +153 -0
- package/src/core/plugins/weather/weather-middleware.ts +27 -0
- package/src/core/plugins/weather/weather-sub-agent.ts +65 -0
- package/src/core/plugins/weather/weather-tools.ts +103 -0
- package/src/core/plugins/weather/weather.plugin.ts +227 -0
- package/src/core/prompt-composer.ts +469 -0
- package/src/core/registries.ts +642 -0
- package/src/core/runtime-context.ts +619 -0
- package/src/core/state.ts +108 -0
- package/src/core/sub-agent-fallback.ts +187 -0
- package/src/core/subagent-as-tool.ts +322 -0
- package/src/core/template.ts +245 -0
- package/src/core/utils.ts +246 -0
- package/src/core/wrap-plugin-tool.ts +100 -0
- package/src/do/ambient.ts +266 -0
- package/src/do/contracts.ts +562 -0
- package/src/do/debug-timers.ts +120 -0
- package/src/do/fresh-stub.ts +32 -0
- package/src/do/gateway-retry.ts +73 -0
- package/src/do/secrets-adapter.ts +32 -0
- package/src/do/size-alerts.ts +76 -0
- package/src/do/sse-stream.ts +472 -0
- package/src/do/transcript.ts +192 -0
- package/src/do/turn-body.ts +84 -0
- package/src/do/turn-metadata.ts +146 -0
- package/src/do/ucan-service.ts +339 -0
- package/src/do/user-oracle-do.ts +3035 -0
- package/src/gateway-worker.ts +82 -0
- package/src/index.ts +237 -0
- package/src/llm/byo-adapter.ts +59 -0
- package/src/llm/byo-catalog.ts +520 -0
- package/src/llm/byo-client.ts +280 -0
- package/src/llm/byo-routes.ts +255 -0
- package/src/llm/byo-service.ts +836 -0
- package/src/llm/chatgpt-oauth.ts +430 -0
- package/src/llm/provider-error.ts +283 -0
- package/src/matrix/gateway-do.ts +918 -0
- package/src/matrix/ingest.ts +209 -0
- package/src/matrix/replay-format.ts +34 -0
- package/src/matrix/reply-chain.ts +109 -0
- package/src/matrix/room-state-codec.ts +115 -0
- package/src/matrix/user-homeserver.ts +106 -0
- package/src/matrix/work-status.ts +294 -0
- package/src/memory/session-history-indexer.ts +328 -0
- package/src/owner-store/ixo-vfs-store.ts +570 -0
- package/src/owner-store/matrix-media-store.ts +136 -0
- package/src/owner-store/migrating-store.ts +185 -0
- package/src/owner-store/owner-copy-errors.ts +175 -0
- package/src/owner-store/retry.ts +45 -0
- package/src/owner-store/tus-upload.ts +333 -0
- package/src/owner-store/types.ts +0 -0
- package/src/plugin-api/define-plugin.ts +45 -0
- package/src/plugin-api/index.ts +56 -0
- package/src/plugin-api/oracle-plugin.ts +145 -0
- package/src/plugin-api/tool-helper.ts +82 -0
- package/src/plugin-api/types.ts +816 -0
- package/src/plugin-api/ucan-errors.ts +16 -0
- package/src/plugins/agui/agui-agent.ts +115 -0
- package/src/plugins/agui/agui.plugin.ts +148 -0
- package/src/plugins/agui/index.ts +2 -0
- package/src/plugins/attachments/attachments.plugin.ts +54 -0
- package/src/plugins/attachments/index.ts +6 -0
- package/src/plugins/attachments/view-attachment-tool.ts +91 -0
- package/src/plugins/base64.ts +26 -0
- package/src/plugins/composio/composio-tools.ts +365 -0
- package/src/plugins/composio/composio-ucan.ts +43 -0
- package/src/plugins/composio/composio.plugin.ts +214 -0
- package/src/plugins/composio/index.ts +15 -0
- package/src/plugins/domain-indexer/domain-indexer-agent.ts +57 -0
- package/src/plugins/domain-indexer/domain-indexer-tools.ts +253 -0
- package/src/plugins/domain-indexer/domain-indexer.plugin.ts +100 -0
- package/src/plugins/domain-indexer/index.ts +3 -0
- package/src/plugins/editor/blocknote-bridge.ts +120 -0
- package/src/plugins/editor/content-session.ts +331 -0
- package/src/plugins/editor/content-tools.ts +686 -0
- package/src/plugins/editor/document-model.ts +0 -0
- package/src/plugins/editor/editor-access-denied-tool.ts +58 -0
- package/src/plugins/editor/editor-agent.ts +34 -0
- package/src/plugins/editor/editor-config.ts +64 -0
- package/src/plugins/editor/editor-mx.ts +128 -0
- package/src/plugins/editor/editor.plugin.ts +143 -0
- package/src/plugins/editor/failures.ts +154 -0
- package/src/plugins/editor/index.ts +31 -0
- package/src/plugins/editor/jsdom-shim.ts +55 -0
- package/src/plugins/editor/prompts.ts +59 -0
- package/src/plugins/editor/prop-policy.ts +147 -0
- package/src/plugins/editor/provider.ts +386 -0
- package/src/plugins/editor/room-membership.ts +95 -0
- package/src/plugins/editor/standalone-editor-tool.ts +200 -0
- package/src/plugins/firecrawl/firecrawl-agent.ts +93 -0
- package/src/plugins/firecrawl/firecrawl-tools.ts +220 -0
- package/src/plugins/firecrawl/firecrawl.plugin.ts +105 -0
- package/src/plugins/firecrawl/index.ts +15 -0
- package/src/plugins/flows/action-metadata.ts +3505 -0
- package/src/plugins/flows/actions.ts +190 -0
- package/src/plugins/flows/edit.ts +314 -0
- package/src/plugins/flows/errors.ts +38 -0
- package/src/plugins/flows/explain.ts +43 -0
- package/src/plugins/flows/flow-doc.ts +111 -0
- package/src/plugins/flows/flows.plugin.ts +129 -0
- package/src/plugins/flows/forms.ts +330 -0
- package/src/plugins/flows/index.ts +2 -0
- package/src/plugins/flows/linkage.ts +147 -0
- package/src/plugins/flows/port-types.ts +53 -0
- package/src/plugins/flows/prompts.ts +65 -0
- package/src/plugins/flows/read.ts +392 -0
- package/src/plugins/flows/references.ts +66 -0
- package/src/plugins/flows/test-support.ts +165 -0
- package/src/plugins/flows/tools/authoring.ts +516 -0
- package/src/plugins/flows/tools/discovery.ts +119 -0
- package/src/plugins/flows/tools/forms.ts +142 -0
- package/src/plugins/flows/tools/inspect.ts +165 -0
- package/src/plugins/flows/tools/linkage.ts +117 -0
- package/src/plugins/flows/tools/settings.ts +184 -0
- package/src/plugins/flows/translator.ts +328 -0
- package/src/plugins/flows/types.ts +210 -0
- package/src/plugins/flows/ydoc-helpers.ts +543 -0
- package/src/plugins/index.ts +70 -0
- package/src/plugins/mcp-call-timeout.ts +38 -0
- package/src/plugins/mcp-tool-adapter.ts +87 -0
- package/src/plugins/memory/index.ts +17 -0
- package/src/plugins/memory/memory-tools.ts +463 -0
- package/src/plugins/memory/memory-ucan.ts +51 -0
- package/src/plugins/memory/memory.plugin.ts +122 -0
- package/src/plugins/memory/types.ts +147 -0
- package/src/plugins/portal/action-log.ts +34 -0
- package/src/plugins/portal/index.ts +2 -0
- package/src/plugins/portal/portal.plugin.ts +203 -0
- package/src/plugins/sandbox/index.ts +34 -0
- package/src/plugins/sandbox/sandbox-bridge.ts +295 -0
- package/src/plugins/sandbox/sandbox-mcp.ts +127 -0
- package/src/plugins/sandbox/sandbox-write-blob.ts +103 -0
- package/src/plugins/sandbox/sandbox.plugin.ts +599 -0
- package/src/plugins/tasks/index.ts +17 -0
- package/src/plugins/tasks/manifest.ts +81 -0
- package/src/plugins/tasks/middleware.ts +219 -0
- package/src/plugins/tasks/tasks-tools.ts +414 -0
- package/src/plugins/tasks/tasks.plugin.ts +74 -0
- package/src/plugins/ucan-failure.ts +133 -0
- package/src/plugins/user-preferences/index.ts +22 -0
- package/src/plugins/user-preferences/schema.ts +31 -0
- package/src/plugins/user-preferences/user-preferences-store.ts +172 -0
- package/src/plugins/user-preferences/user-preferences-tool.ts +131 -0
- package/src/plugins/user-preferences/user-preferences.plugin.ts +137 -0
- package/src/plugins/vfs/index.ts +40 -0
- package/src/plugins/vfs/vfs-auth.ts +56 -0
- package/src/plugins/vfs/vfs-client.ts +655 -0
- package/src/plugins/vfs/vfs-content.ts +181 -0
- package/src/plugins/vfs/vfs-errors.ts +163 -0
- package/src/plugins/vfs/vfs-sandbox-tools.ts +272 -0
- package/src/plugins/vfs/vfs-tools.ts +635 -0
- package/src/plugins/vfs/vfs.plugin.ts +201 -0
- package/src/realtime/frontend-call-registry.ts +163 -0
- package/src/realtime/realtime-endpoint.ts +619 -0
- package/src/realtime/session-socket-hub.ts +249 -0
- package/src/realtime/socket-io-codec.ts +187 -0
- package/src/secrets/jwe.ts +74 -0
- package/src/secrets/pin-cipher.ts +43 -0
- package/src/secrets/secrets-service.ts +319 -0
- package/src/shell/app.ts +564 -0
- package/src/shell/auth.ts +290 -0
- package/src/sqlite/blob-codec.ts +100 -0
- package/src/sqlite/blob-compactor.ts +146 -0
- package/src/sqlite/cache-config.ts +45 -0
- package/src/sqlite/database.ts +643 -0
- package/src/sqlite/do-vfs.ts +1485 -0
- package/src/sqlite/index.ts +61 -0
- package/src/sqlite/serialization.ts +370 -0
- package/src/sqlite/sessions-store.ts +311 -0
- package/src/sqlite/sqlite-saver.ts +919 -0
- package/src/sqlite/vacuum-policy.ts +77 -0
- package/src/sqlite/wa-sqlite-loader.ts +123 -0
- package/src/sqlite/wasm.d.ts +10 -0
- package/src/tasks/schedule.ts +174 -0
- package/src/tasks/scheduler.ts +753 -0
- package/src/tasks/spec.ts +125 -0
- package/src/tasks/store.ts +351 -0
|
@@ -0,0 +1,919 @@
|
|
|
1
|
+
/* eslint-disable no-console -- console replaces the Node runtime's @ixo/logger Logger on workerd */
|
|
2
|
+
/**
|
|
3
|
+
* LangGraph checkpointer over `DoSqliteDatabase` — a port of the Node
|
|
4
|
+
* runtime's `@ixo/sqlite-saver` (better-sqlite3) with the SAME schema and
|
|
5
|
+
* semantics. Compatibility is one-way: a `.db` written by the Node runtime
|
|
6
|
+
* opens here unchanged, but blobs THIS runtime writes are gzipped
|
|
7
|
+
* (`blob-codec.ts` — lossless, detected by magic bytes on read), which the
|
|
8
|
+
* Node runtime does not understand. Schema:
|
|
9
|
+
*
|
|
10
|
+
* checkpoints(thread_id, checkpoint_ns, checkpoint_id, parent_checkpoint_id,
|
|
11
|
+
* type, checkpoint BLOB, metadata BLOB)
|
|
12
|
+
* writes(thread_id, checkpoint_ns, checkpoint_id, task_id, idx, channel,
|
|
13
|
+
* type, value BLOB)
|
|
14
|
+
* messages(thread_id, checkpoint_ns, checkpoint_id, message_id PK,
|
|
15
|
+
* message_type, message_content, message BLOB, created_at)
|
|
16
|
+
* schema_migrations(version PK, name, applied_at)
|
|
17
|
+
*
|
|
18
|
+
* Behaviour kept from the original:
|
|
19
|
+
* - `channel_values.messages` is stripped from the stored checkpoint and each
|
|
20
|
+
* message is upserted into `messages` (keyed by message id, so the table is
|
|
21
|
+
* the full transcript: rows survive checkpoint pruning and summarization).
|
|
22
|
+
* - `put` prunes a thread to `maxCheckpointsPerThread` newest checkpoints (and
|
|
23
|
+
* their writes) once it exceeds the cap by `PRUNE_SLACK`.
|
|
24
|
+
* - `pending_sends` migration for pre-v4 checkpoints.
|
|
25
|
+
* - Serialization is `JsonPlusSerializer` (the `BaseCheckpointSaver` default).
|
|
26
|
+
*
|
|
27
|
+
* Differences: everything is async (wa-sqlite API), `console` replaces the
|
|
28
|
+
* `@ixo/logger` Logger, the oracle name stamped on messages comes from
|
|
29
|
+
* `options.oracleName` instead of `process.env`, and each `put`/`putWrites`/
|
|
30
|
+
* `deleteThread` runs inside a `DoSqliteDatabase.transaction` — which the VFS
|
|
31
|
+
* commits to DO storage atomically.
|
|
32
|
+
*/
|
|
33
|
+
import type { BaseMessage } from '@langchain/core/messages';
|
|
34
|
+
import type { RunnableConfig } from '@langchain/core/runnables';
|
|
35
|
+
import {
|
|
36
|
+
BaseCheckpointSaver,
|
|
37
|
+
type Checkpoint,
|
|
38
|
+
type CheckpointListOptions,
|
|
39
|
+
type CheckpointMetadata,
|
|
40
|
+
type CheckpointTuple,
|
|
41
|
+
copyCheckpoint,
|
|
42
|
+
maxChannelVersion,
|
|
43
|
+
type PendingWrite,
|
|
44
|
+
type SerializerProtocol,
|
|
45
|
+
TASKS,
|
|
46
|
+
} from '@langchain/langgraph-checkpoint';
|
|
47
|
+
import { compressForStorage, hexToBytes, readBlobText } from './blob-codec';
|
|
48
|
+
import type { DoSqliteDatabase, SqlParam } from './database';
|
|
49
|
+
import {
|
|
50
|
+
_default,
|
|
51
|
+
cleanAdditionalKwargs,
|
|
52
|
+
stringify,
|
|
53
|
+
type CleanAdditionalKwargs,
|
|
54
|
+
} from './serialization';
|
|
55
|
+
|
|
56
|
+
export interface SqliteSaverOptions {
|
|
57
|
+
/**
|
|
58
|
+
* Newest checkpoints retained per thread after each `put`. LangGraph writes
|
|
59
|
+
* one checkpoint per super-step and never deletes them; without a cap a
|
|
60
|
+
* long-lived thread's tables grow without bound. `0` disables pruning.
|
|
61
|
+
*/
|
|
62
|
+
maxCheckpointsPerThread?: number;
|
|
63
|
+
/** Stamped into every persisted message's `additional_kwargs.oracleName`. */
|
|
64
|
+
oracleName?: string;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Checkpoints kept per thread. Only the newest is read when a turn starts;
|
|
69
|
+
* older ones exist for replay/time-travel (unused today) and for recovering
|
|
70
|
+
* an interrupted step. Ten (the Node default is twenty) roughly halves the
|
|
71
|
+
* two largest tables of a busy user's file (`checkpoints` + `writes`); raise
|
|
72
|
+
* it again once the per-object 10 GB ceiling is no longer the constraint.
|
|
73
|
+
*/
|
|
74
|
+
export const DEFAULT_MAX_CHECKPOINTS_PER_THREAD = 10;
|
|
75
|
+
|
|
76
|
+
/** Pruning only runs once a thread exceeds the cap by this many checkpoints. */
|
|
77
|
+
export const PRUNE_SLACK = 5;
|
|
78
|
+
|
|
79
|
+
const DEFAULT_ORACLE_NAME = 'IXO Oracle';
|
|
80
|
+
|
|
81
|
+
type CheckpointRow = {
|
|
82
|
+
thread_id: string;
|
|
83
|
+
checkpoint_ns: string;
|
|
84
|
+
checkpoint_id: string;
|
|
85
|
+
parent_checkpoint_id: string | null;
|
|
86
|
+
type: string | null;
|
|
87
|
+
checkpoint: Uint8Array | string;
|
|
88
|
+
metadata: Uint8Array | string;
|
|
89
|
+
pending_writes: string;
|
|
90
|
+
pending_sends: string;
|
|
91
|
+
};
|
|
92
|
+
|
|
93
|
+
type MessageRow = {
|
|
94
|
+
message: Uint8Array | string;
|
|
95
|
+
};
|
|
96
|
+
|
|
97
|
+
interface PendingWriteColumn {
|
|
98
|
+
task_id: string;
|
|
99
|
+
channel: string;
|
|
100
|
+
type: string | null;
|
|
101
|
+
/** `hex(value)` — decoded via {@link hexColumnText}. */
|
|
102
|
+
value: string | null;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
interface PendingSendColumn {
|
|
106
|
+
type: string;
|
|
107
|
+
/** `hex(value)` — decoded via {@link hexColumnText}. */
|
|
108
|
+
value: string | null;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
interface Migration {
|
|
112
|
+
version: number;
|
|
113
|
+
name: string;
|
|
114
|
+
up: (db: DoSqliteDatabase) => Promise<void>;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
type ChannelValues = Record<string, unknown>;
|
|
118
|
+
|
|
119
|
+
interface CheckpointWithMessages extends Checkpoint {
|
|
120
|
+
channel_values: ChannelValues & { messages?: BaseMessage[] };
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* Compile-time guard: the metadata keys `list()` accepts as filters must stay
|
|
125
|
+
* in sync with `CheckpointMetadata`. If this stops compiling, update the list.
|
|
126
|
+
*/
|
|
127
|
+
const checkpointMetadataKeys = ['source', 'step', 'parents'] as const;
|
|
128
|
+
type MetadataKey = keyof CheckpointMetadata;
|
|
129
|
+
type CheckKeys<K extends readonly MetadataKey[]> = [MetadataKey] extends [
|
|
130
|
+
K[number],
|
|
131
|
+
]
|
|
132
|
+
? K
|
|
133
|
+
: never;
|
|
134
|
+
const validCheckpointMetadataKeys: readonly string[] = ((
|
|
135
|
+
keys: CheckKeys<typeof checkpointMetadataKeys>,
|
|
136
|
+
) => keys)(checkpointMetadataKeys);
|
|
137
|
+
|
|
138
|
+
/** Migration 001: `messages.created_at` (older files lack it) plus its index. */
|
|
139
|
+
const migration001: Migration = {
|
|
140
|
+
version: 1,
|
|
141
|
+
name: 'add_created_at_to_messages',
|
|
142
|
+
up: async (db) => {
|
|
143
|
+
const columns = await db.exec<{ name: string }>(
|
|
144
|
+
`SELECT name FROM pragma_table_info('messages') WHERE name = ?`,
|
|
145
|
+
['created_at'],
|
|
146
|
+
);
|
|
147
|
+
if (columns.length === 0) {
|
|
148
|
+
await db.run(
|
|
149
|
+
`ALTER TABLE messages ADD COLUMN created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP`,
|
|
150
|
+
);
|
|
151
|
+
await db.run(
|
|
152
|
+
`UPDATE messages SET created_at = CURRENT_TIMESTAMP WHERE created_at IS NULL`,
|
|
153
|
+
);
|
|
154
|
+
}
|
|
155
|
+
const index = await db.get<{ name: string }>(
|
|
156
|
+
`SELECT name FROM sqlite_master WHERE type = 'index' AND name = 'idx_messages_thread_created'`,
|
|
157
|
+
);
|
|
158
|
+
if (index === undefined) {
|
|
159
|
+
await db.run(
|
|
160
|
+
`CREATE INDEX idx_messages_thread_created ON messages(thread_id, created_at)`,
|
|
161
|
+
);
|
|
162
|
+
}
|
|
163
|
+
},
|
|
164
|
+
};
|
|
165
|
+
|
|
166
|
+
const MIGRATIONS: readonly Migration[] = [migration001];
|
|
167
|
+
|
|
168
|
+
// `writes.value` blobs may be gzipped, so they travel as hex() — a
|
|
169
|
+
// CAST(... AS TEXT) of compressed bytes would mangle them irreversibly.
|
|
170
|
+
const TUPLE_SELECT = `
|
|
171
|
+
SELECT
|
|
172
|
+
thread_id,
|
|
173
|
+
checkpoint_ns,
|
|
174
|
+
checkpoint_id,
|
|
175
|
+
parent_checkpoint_id,
|
|
176
|
+
type,
|
|
177
|
+
checkpoint,
|
|
178
|
+
metadata,
|
|
179
|
+
(
|
|
180
|
+
SELECT json_group_array(
|
|
181
|
+
json_object('task_id', pw.task_id, 'channel', pw.channel, 'type', pw.type, 'value', hex(pw.value))
|
|
182
|
+
)
|
|
183
|
+
FROM writes AS pw
|
|
184
|
+
WHERE pw.thread_id = checkpoints.thread_id
|
|
185
|
+
AND pw.checkpoint_ns = checkpoints.checkpoint_ns
|
|
186
|
+
AND pw.checkpoint_id = checkpoints.checkpoint_id
|
|
187
|
+
) AS pending_writes,
|
|
188
|
+
(
|
|
189
|
+
SELECT json_group_array(json_object('type', ps.type, 'value', hex(ps.value)))
|
|
190
|
+
FROM writes AS ps
|
|
191
|
+
WHERE ps.thread_id = checkpoints.thread_id
|
|
192
|
+
AND ps.checkpoint_ns = checkpoints.checkpoint_ns
|
|
193
|
+
AND ps.checkpoint_id = checkpoints.parent_checkpoint_id
|
|
194
|
+
AND ps.channel = '${TASKS}'
|
|
195
|
+
ORDER BY ps.idx
|
|
196
|
+
) AS pending_sends
|
|
197
|
+
FROM checkpoints`;
|
|
198
|
+
|
|
199
|
+
/**
|
|
200
|
+
* A `hex(column)` value from a tuple query back to its JSON text. `hex(NULL)`
|
|
201
|
+
* yields an empty string; codec-written blobs gunzip transparently.
|
|
202
|
+
*/
|
|
203
|
+
async function hexColumnText(hex: string | null): Promise<string> {
|
|
204
|
+
if (hex === null || hex.length === 0) return '';
|
|
205
|
+
return readBlobText(hexToBytes(hex));
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
/** `LIKE` pattern matching rows whose column contains `needle` literally. */
|
|
209
|
+
function likeContains(needle: string): string {
|
|
210
|
+
return `%${needle.replace(/[\\%_]/g, (c) => `\\${c}`)}%`;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
export class SqliteSaver extends BaseCheckpointSaver {
|
|
214
|
+
readonly db: DoSqliteDatabase;
|
|
215
|
+
protected readonly maxCheckpointsPerThread: number;
|
|
216
|
+
protected readonly oracleName: string;
|
|
217
|
+
private setupPromise: Promise<void> | undefined;
|
|
218
|
+
|
|
219
|
+
constructor(
|
|
220
|
+
db: DoSqliteDatabase,
|
|
221
|
+
serde?: SerializerProtocol,
|
|
222
|
+
options: SqliteSaverOptions = {},
|
|
223
|
+
) {
|
|
224
|
+
super(serde);
|
|
225
|
+
this.db = db;
|
|
226
|
+
this.maxCheckpointsPerThread =
|
|
227
|
+
options.maxCheckpointsPerThread ?? DEFAULT_MAX_CHECKPOINTS_PER_THREAD;
|
|
228
|
+
this.oracleName = options.oracleName ?? DEFAULT_ORACLE_NAME;
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
/** Create tables/indexes and run pending migrations. Idempotent; concurrent callers share one run. */
|
|
232
|
+
setup(): Promise<void> {
|
|
233
|
+
this.setupPromise ??= this.doSetup().catch((error: unknown) => {
|
|
234
|
+
this.setupPromise = undefined;
|
|
235
|
+
throw error;
|
|
236
|
+
});
|
|
237
|
+
return this.setupPromise;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
private async doSetup(): Promise<void> {
|
|
241
|
+
const db = this.db;
|
|
242
|
+
// Incremental auto-vacuum lets pruning hand freed pages back so the file
|
|
243
|
+
// (what gets exported to the owner store) doesn't sit at its high-water
|
|
244
|
+
// mark. Binds immediately on a new database (runs before the first table);
|
|
245
|
+
// on an existing NONE-mode file it needs a full VACUUM, done elsewhere.
|
|
246
|
+
const autoVacuum = await db.get<{ auto_vacuum: number }>(
|
|
247
|
+
'PRAGMA auto_vacuum',
|
|
248
|
+
);
|
|
249
|
+
if (autoVacuum !== undefined && autoVacuum.auto_vacuum !== 2) {
|
|
250
|
+
await db.run('PRAGMA auto_vacuum = INCREMENTAL');
|
|
251
|
+
}
|
|
252
|
+
await db.run(`
|
|
253
|
+
CREATE TABLE IF NOT EXISTS checkpoints (
|
|
254
|
+
thread_id TEXT NOT NULL,
|
|
255
|
+
checkpoint_ns TEXT NOT NULL DEFAULT '',
|
|
256
|
+
checkpoint_id TEXT NOT NULL,
|
|
257
|
+
parent_checkpoint_id TEXT,
|
|
258
|
+
type TEXT,
|
|
259
|
+
checkpoint BLOB,
|
|
260
|
+
metadata BLOB,
|
|
261
|
+
PRIMARY KEY (thread_id, checkpoint_ns, checkpoint_id)
|
|
262
|
+
)`);
|
|
263
|
+
await db.run(`
|
|
264
|
+
CREATE TABLE IF NOT EXISTS writes (
|
|
265
|
+
thread_id TEXT NOT NULL,
|
|
266
|
+
checkpoint_ns TEXT NOT NULL DEFAULT '',
|
|
267
|
+
checkpoint_id TEXT NOT NULL,
|
|
268
|
+
task_id TEXT NOT NULL,
|
|
269
|
+
idx INTEGER NOT NULL,
|
|
270
|
+
channel TEXT NOT NULL,
|
|
271
|
+
type TEXT,
|
|
272
|
+
value BLOB,
|
|
273
|
+
PRIMARY KEY (thread_id, checkpoint_ns, checkpoint_id, task_id, idx)
|
|
274
|
+
)`);
|
|
275
|
+
await db.run(`
|
|
276
|
+
CREATE TABLE IF NOT EXISTS messages (
|
|
277
|
+
thread_id TEXT NOT NULL,
|
|
278
|
+
checkpoint_ns TEXT NOT NULL DEFAULT '',
|
|
279
|
+
checkpoint_id TEXT NOT NULL,
|
|
280
|
+
message_id TEXT NOT NULL,
|
|
281
|
+
message_type TEXT NOT NULL,
|
|
282
|
+
message_content TEXT NOT NULL,
|
|
283
|
+
message BLOB,
|
|
284
|
+
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
285
|
+
PRIMARY KEY (message_id)
|
|
286
|
+
)`);
|
|
287
|
+
await this.runMigrations();
|
|
288
|
+
await db.run(
|
|
289
|
+
`CREATE INDEX IF NOT EXISTS idx_messages_thread_id ON messages(thread_id)`,
|
|
290
|
+
);
|
|
291
|
+
await db.run(
|
|
292
|
+
`CREATE INDEX IF NOT EXISTS idx_messages_checkpoint_id ON messages(checkpoint_id)`,
|
|
293
|
+
);
|
|
294
|
+
await db.run(
|
|
295
|
+
`CREATE INDEX IF NOT EXISTS idx_messages_lookup ON messages(thread_id, checkpoint_ns, checkpoint_id)`,
|
|
296
|
+
);
|
|
297
|
+
await db.run(
|
|
298
|
+
`CREATE INDEX IF NOT EXISTS idx_messages_thread_created ON messages(thread_id, created_at)`,
|
|
299
|
+
);
|
|
300
|
+
await db.run(
|
|
301
|
+
`CREATE INDEX IF NOT EXISTS idx_writes_channel ON writes(thread_id, checkpoint_id, channel)`,
|
|
302
|
+
);
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
private async runMigrations(): Promise<void> {
|
|
306
|
+
const db = this.db;
|
|
307
|
+
await db.run(`
|
|
308
|
+
CREATE TABLE IF NOT EXISTS schema_migrations (
|
|
309
|
+
version INTEGER NOT NULL PRIMARY KEY,
|
|
310
|
+
name TEXT NOT NULL,
|
|
311
|
+
applied_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
312
|
+
)`);
|
|
313
|
+
const applied = new Set(
|
|
314
|
+
(
|
|
315
|
+
await db.exec<{ version: number }>(
|
|
316
|
+
'SELECT version FROM schema_migrations ORDER BY version',
|
|
317
|
+
)
|
|
318
|
+
).map((row) => row.version),
|
|
319
|
+
);
|
|
320
|
+
const pending = MIGRATIONS.filter((m) => !applied.has(m.version)).sort(
|
|
321
|
+
(a, b) => a.version - b.version,
|
|
322
|
+
);
|
|
323
|
+
if (pending.length === 0) return;
|
|
324
|
+
console.info(
|
|
325
|
+
`[sqlite-saver] running ${pending.length} pending migration(s)`,
|
|
326
|
+
);
|
|
327
|
+
for (const migration of pending) {
|
|
328
|
+
try {
|
|
329
|
+
await db.transaction(async () => {
|
|
330
|
+
await migration.up(db);
|
|
331
|
+
await db.run(
|
|
332
|
+
'INSERT INTO schema_migrations (version, name, applied_at) VALUES (?, ?, CURRENT_TIMESTAMP)',
|
|
333
|
+
[migration.version, migration.name],
|
|
334
|
+
);
|
|
335
|
+
});
|
|
336
|
+
console.info(
|
|
337
|
+
`[sqlite-saver] migration ${migration.version}: ${migration.name} applied`,
|
|
338
|
+
);
|
|
339
|
+
} catch (error) {
|
|
340
|
+
console.error(
|
|
341
|
+
`[sqlite-saver] migration ${migration.version}: ${migration.name} failed`,
|
|
342
|
+
error,
|
|
343
|
+
);
|
|
344
|
+
throw error;
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
/**
|
|
350
|
+
* Every message ever written for a thread, oldest first, independent of
|
|
351
|
+
* which checkpoint currently references it (the full transcript, even after
|
|
352
|
+
* summarization condensed the graph state).
|
|
353
|
+
*/
|
|
354
|
+
async listThreadMessages(threadId: string): Promise<BaseMessage[]> {
|
|
355
|
+
await this.setup();
|
|
356
|
+
const rows = await this.db.exec<MessageRow>(
|
|
357
|
+
'SELECT message FROM messages WHERE thread_id = ? ORDER BY created_at ASC, rowid ASC',
|
|
358
|
+
[threadId],
|
|
359
|
+
);
|
|
360
|
+
return Promise.all(rows.map((row) => this.loadMessage(row.message)));
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
/**
|
|
364
|
+
* The thread's messages whose stored plain content (`message_content`)
|
|
365
|
+
* contains `contentContains`, oldest first — a SQL `LIKE` pre-filter, so
|
|
366
|
+
* only the matching rows' blobs are inflated. Attachment payload retention
|
|
367
|
+
* uses it to find rows that still hold inline bytes / a placeholder.
|
|
368
|
+
*/
|
|
369
|
+
async listThreadMessagesMatching(
|
|
370
|
+
threadId: string,
|
|
371
|
+
filter: { messageType?: string; contentContains: string },
|
|
372
|
+
): Promise<BaseMessage[]> {
|
|
373
|
+
await this.setup();
|
|
374
|
+
const rows = await this.db.exec<MessageRow>(
|
|
375
|
+
`SELECT message FROM messages WHERE thread_id = ?${
|
|
376
|
+
filter.messageType !== undefined ? ' AND message_type = ?' : ''
|
|
377
|
+
} AND message_content LIKE ? ESCAPE '\\' ORDER BY created_at ASC, rowid ASC`,
|
|
378
|
+
[
|
|
379
|
+
threadId,
|
|
380
|
+
...(filter.messageType !== undefined ? [filter.messageType] : []),
|
|
381
|
+
likeContains(filter.contentContains),
|
|
382
|
+
],
|
|
383
|
+
);
|
|
384
|
+
return Promise.all(rows.map((row) => this.loadMessage(row.message)));
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
/** Whether `listThreadMessagesMatching` would return anything (no blob inflation). */
|
|
388
|
+
async hasThreadMessageMatching(
|
|
389
|
+
threadId: string,
|
|
390
|
+
filter: { messageType?: string; contentContains: string },
|
|
391
|
+
): Promise<boolean> {
|
|
392
|
+
await this.setup();
|
|
393
|
+
const rows = await this.db.exec<{ found: number }>(
|
|
394
|
+
`SELECT 1 AS found FROM messages WHERE thread_id = ?${
|
|
395
|
+
filter.messageType !== undefined ? ' AND message_type = ?' : ''
|
|
396
|
+
} AND message_content LIKE ? ESCAPE '\\' LIMIT 1`,
|
|
397
|
+
[
|
|
398
|
+
threadId,
|
|
399
|
+
...(filter.messageType !== undefined ? [filter.messageType] : []),
|
|
400
|
+
likeContains(filter.contentContains),
|
|
401
|
+
],
|
|
402
|
+
);
|
|
403
|
+
return rows.length > 0;
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
/**
|
|
407
|
+
* Rewrite one persisted message in place — same row, checkpoint linkage
|
|
408
|
+
* and `created_at` (transcript order) untouched. Returns false when the
|
|
409
|
+
* thread has no row for the message's id. Attachment payload retention
|
|
410
|
+
* uses it to swap inline bytes for a placeholder after a turn.
|
|
411
|
+
*/
|
|
412
|
+
async replaceThreadMessage(
|
|
413
|
+
threadId: string,
|
|
414
|
+
message: BaseMessage,
|
|
415
|
+
): Promise<boolean> {
|
|
416
|
+
await this.setup();
|
|
417
|
+
// Only the type/content/blob columns of the row are taken from the
|
|
418
|
+
// encoder; the checkpoint columns it also produces stay as stored.
|
|
419
|
+
const [, , , messageId, messageType, messageContent, blob] =
|
|
420
|
+
await this.toMessageRow(message, threadId, '', '');
|
|
421
|
+
const result = await this.db.run(
|
|
422
|
+
'UPDATE messages SET message_type = ?, message_content = ?, message = ? WHERE thread_id = ? AND message_id = ?',
|
|
423
|
+
[messageType!, messageContent!, blob!, threadId, messageId!],
|
|
424
|
+
);
|
|
425
|
+
return result.changes > 0;
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
private async loadMessage(raw: Uint8Array | string): Promise<BaseMessage> {
|
|
429
|
+
return (await this.serde.loadsTyped(
|
|
430
|
+
'json',
|
|
431
|
+
await readBlobText(raw),
|
|
432
|
+
)) as BaseMessage;
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
async getTuple(config: RunnableConfig): Promise<CheckpointTuple | undefined> {
|
|
436
|
+
return this.loadTuple(config, true);
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
/**
|
|
440
|
+
* Like `getTuple` but skips the `messages` join and its per-message
|
|
441
|
+
* deserialization — O(1) in conversation length. The returned checkpoint's
|
|
442
|
+
* `channel_values.messages` is left unset.
|
|
443
|
+
*/
|
|
444
|
+
async getTupleWithoutMessages(
|
|
445
|
+
config: RunnableConfig,
|
|
446
|
+
): Promise<CheckpointTuple | undefined> {
|
|
447
|
+
return this.loadTuple(config, false);
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
protected async loadTuple(
|
|
451
|
+
config: RunnableConfig,
|
|
452
|
+
includeMessages: boolean,
|
|
453
|
+
): Promise<CheckpointTuple | undefined> {
|
|
454
|
+
await this.setup();
|
|
455
|
+
const configurable = config.configurable ?? {};
|
|
456
|
+
const threadId: unknown = configurable.thread_id;
|
|
457
|
+
const checkpointNs: unknown = configurable.checkpoint_ns ?? '';
|
|
458
|
+
const checkpointId: unknown = configurable.checkpoint_id;
|
|
459
|
+
if (typeof threadId !== 'string' || typeof checkpointNs !== 'string')
|
|
460
|
+
return undefined;
|
|
461
|
+
|
|
462
|
+
const row =
|
|
463
|
+
typeof checkpointId === 'string' && checkpointId.length > 0
|
|
464
|
+
? await this.db.get<CheckpointRow>(
|
|
465
|
+
`${TUPLE_SELECT} WHERE thread_id = ? AND checkpoint_ns = ? AND checkpoint_id = ?`,
|
|
466
|
+
[threadId, checkpointNs, checkpointId],
|
|
467
|
+
)
|
|
468
|
+
: await this.db.get<CheckpointRow>(
|
|
469
|
+
`${TUPLE_SELECT} WHERE thread_id = ? AND checkpoint_ns = ? ORDER BY checkpoint_id DESC LIMIT 1`,
|
|
470
|
+
[threadId, checkpointNs],
|
|
471
|
+
);
|
|
472
|
+
if (row === undefined) return undefined;
|
|
473
|
+
|
|
474
|
+
const finalConfig: RunnableConfig =
|
|
475
|
+
typeof checkpointId === 'string' && checkpointId.length > 0
|
|
476
|
+
? config
|
|
477
|
+
: {
|
|
478
|
+
configurable: {
|
|
479
|
+
thread_id: row.thread_id,
|
|
480
|
+
checkpoint_ns: checkpointNs,
|
|
481
|
+
checkpoint_id: row.checkpoint_id,
|
|
482
|
+
},
|
|
483
|
+
};
|
|
484
|
+
|
|
485
|
+
const tuple = await this.rowToTuple(row, includeMessages, finalConfig);
|
|
486
|
+
return tuple;
|
|
487
|
+
}
|
|
488
|
+
|
|
489
|
+
private async rowToTuple(
|
|
490
|
+
row: CheckpointRow,
|
|
491
|
+
includeMessages: boolean,
|
|
492
|
+
config: RunnableConfig,
|
|
493
|
+
): Promise<CheckpointTuple> {
|
|
494
|
+
const pendingWrites = await this.loadPendingWrites(row.pending_writes);
|
|
495
|
+
const checkpoint = (await this.serde.loadsTyped(
|
|
496
|
+
row.type ?? 'json',
|
|
497
|
+
await readBlobText(row.checkpoint),
|
|
498
|
+
)) as Checkpoint;
|
|
499
|
+
if (includeMessages) {
|
|
500
|
+
const messages = await this.loadCheckpointMessages(
|
|
501
|
+
row.thread_id,
|
|
502
|
+
row.checkpoint_ns,
|
|
503
|
+
row.checkpoint_id,
|
|
504
|
+
);
|
|
505
|
+
if (messages.length > 0) checkpoint.channel_values.messages = messages;
|
|
506
|
+
}
|
|
507
|
+
if (checkpoint.v < 4 && row.parent_checkpoint_id !== null) {
|
|
508
|
+
await this.migratePendingSends(
|
|
509
|
+
checkpoint,
|
|
510
|
+
row.thread_id,
|
|
511
|
+
row.parent_checkpoint_id,
|
|
512
|
+
);
|
|
513
|
+
}
|
|
514
|
+
const metadata = (await this.serde.loadsTyped(
|
|
515
|
+
row.type ?? 'json',
|
|
516
|
+
await readBlobText(row.metadata),
|
|
517
|
+
)) as CheckpointMetadata;
|
|
518
|
+
return {
|
|
519
|
+
config,
|
|
520
|
+
checkpoint,
|
|
521
|
+
metadata,
|
|
522
|
+
parentConfig:
|
|
523
|
+
row.parent_checkpoint_id !== null
|
|
524
|
+
? {
|
|
525
|
+
configurable: {
|
|
526
|
+
thread_id: row.thread_id,
|
|
527
|
+
checkpoint_ns: row.checkpoint_ns,
|
|
528
|
+
checkpoint_id: row.parent_checkpoint_id,
|
|
529
|
+
},
|
|
530
|
+
}
|
|
531
|
+
: undefined,
|
|
532
|
+
pendingWrites,
|
|
533
|
+
};
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
private async loadPendingWrites(
|
|
537
|
+
json: string,
|
|
538
|
+
): Promise<Array<[string, string, unknown]>> {
|
|
539
|
+
const columns = JSON.parse(json) as PendingWriteColumn[];
|
|
540
|
+
return Promise.all(
|
|
541
|
+
columns.map(
|
|
542
|
+
async (write): Promise<[string, string, unknown]> => [
|
|
543
|
+
write.task_id,
|
|
544
|
+
write.channel,
|
|
545
|
+
await this.serde.loadsTyped(
|
|
546
|
+
write.type ?? 'json',
|
|
547
|
+
await hexColumnText(write.value),
|
|
548
|
+
),
|
|
549
|
+
],
|
|
550
|
+
),
|
|
551
|
+
);
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
private async loadCheckpointMessages(
|
|
555
|
+
threadId: string,
|
|
556
|
+
checkpointNs: string,
|
|
557
|
+
checkpointId: string,
|
|
558
|
+
): Promise<BaseMessage[]> {
|
|
559
|
+
const rows = await this.db.exec<MessageRow>(
|
|
560
|
+
'SELECT message FROM messages WHERE thread_id = ? AND checkpoint_ns = ? AND checkpoint_id = ?',
|
|
561
|
+
[threadId, checkpointNs, checkpointId],
|
|
562
|
+
);
|
|
563
|
+
return Promise.all(rows.map((row) => this.loadMessage(row.message)));
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
async *list(
|
|
567
|
+
config: RunnableConfig,
|
|
568
|
+
options?: CheckpointListOptions,
|
|
569
|
+
): AsyncGenerator<CheckpointTuple> {
|
|
570
|
+
const { limit, before, filter } = options ?? {};
|
|
571
|
+
await this.setup();
|
|
572
|
+
const threadId: unknown = config.configurable?.thread_id;
|
|
573
|
+
const checkpointNs: unknown = config.configurable?.checkpoint_ns;
|
|
574
|
+
const beforeId: unknown = before?.configurable?.checkpoint_id;
|
|
575
|
+
|
|
576
|
+
const where: string[] = [];
|
|
577
|
+
const args: SqlParam[] = [];
|
|
578
|
+
if (typeof threadId === 'string') {
|
|
579
|
+
where.push('thread_id = ?');
|
|
580
|
+
args.push(threadId);
|
|
581
|
+
}
|
|
582
|
+
if (typeof checkpointNs === 'string') {
|
|
583
|
+
where.push('checkpoint_ns = ?');
|
|
584
|
+
args.push(checkpointNs);
|
|
585
|
+
}
|
|
586
|
+
if (typeof beforeId === 'string') {
|
|
587
|
+
where.push('checkpoint_id < ?');
|
|
588
|
+
args.push(beforeId);
|
|
589
|
+
}
|
|
590
|
+
for (const [key, value] of Object.entries(filter ?? {})) {
|
|
591
|
+
if (value === undefined || !validCheckpointMetadataKeys.includes(key))
|
|
592
|
+
continue;
|
|
593
|
+
// `->` yields the JSON text of the member, compared against the JSON text of the filter value.
|
|
594
|
+
where.push(`json(CAST(metadata AS TEXT))->'$.${key}' = ?`);
|
|
595
|
+
args.push(JSON.stringify(value));
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
let sql = TUPLE_SELECT;
|
|
599
|
+
if (where.length > 0) sql += `\nWHERE ${where.join(' AND ')}`;
|
|
600
|
+
sql += '\nORDER BY checkpoint_id DESC';
|
|
601
|
+
if (limit !== undefined) {
|
|
602
|
+
const parsed = Number.parseInt(String(limit), 10);
|
|
603
|
+
if (Number.isFinite(parsed) && parsed > 0) sql += ` LIMIT ${parsed}`;
|
|
604
|
+
}
|
|
605
|
+
|
|
606
|
+
const rows = await this.db.exec<CheckpointRow>(sql, args);
|
|
607
|
+
for (const row of rows) {
|
|
608
|
+
yield this.rowToTuple(row, true, {
|
|
609
|
+
configurable: {
|
|
610
|
+
thread_id: row.thread_id,
|
|
611
|
+
checkpoint_ns: row.checkpoint_ns,
|
|
612
|
+
checkpoint_id: row.checkpoint_id,
|
|
613
|
+
},
|
|
614
|
+
});
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
|
|
618
|
+
async put(
|
|
619
|
+
config: RunnableConfig,
|
|
620
|
+
checkpointIn: Checkpoint,
|
|
621
|
+
metadata: CheckpointMetadata,
|
|
622
|
+
): Promise<RunnableConfig> {
|
|
623
|
+
await this.setup();
|
|
624
|
+
if (!config.configurable) throw new Error('Empty configuration supplied.');
|
|
625
|
+
const threadId: unknown = config.configurable.thread_id;
|
|
626
|
+
const checkpointNs: unknown = config.configurable.checkpoint_ns ?? '';
|
|
627
|
+
const parentCheckpointId: unknown = config.configurable.checkpoint_id;
|
|
628
|
+
if (typeof threadId !== 'string' || threadId.length === 0) {
|
|
629
|
+
throw new Error(
|
|
630
|
+
`Missing "thread_id" field in passed "config.configurable".`,
|
|
631
|
+
);
|
|
632
|
+
}
|
|
633
|
+
if (typeof checkpointNs !== 'string')
|
|
634
|
+
throw new Error('"checkpoint_ns" must be a string.');
|
|
635
|
+
|
|
636
|
+
const { checkpoint, messages } = removeMessagesFromCheckpoint(checkpointIn);
|
|
637
|
+
const [[type1, serializedCheckpoint], [type2, serializedMetadata]] =
|
|
638
|
+
await Promise.all([
|
|
639
|
+
this.serde.dumpsTyped(checkpoint),
|
|
640
|
+
this.serde.dumpsTyped(metadata),
|
|
641
|
+
]);
|
|
642
|
+
if (type1 !== type2)
|
|
643
|
+
throw new Error(
|
|
644
|
+
'Failed to serialized checkpoint and metadata to the same type.',
|
|
645
|
+
);
|
|
646
|
+
|
|
647
|
+
const messageRows = await Promise.all(
|
|
648
|
+
messages?.map((message) =>
|
|
649
|
+
this.toMessageRow(message, threadId, checkpointNs, checkpoint.id),
|
|
650
|
+
) ?? [],
|
|
651
|
+
);
|
|
652
|
+
// Metadata stays uncompressed: `list()` filters it SQL-side via
|
|
653
|
+
// `json(CAST(metadata AS TEXT))`. It is small (source/step/parents).
|
|
654
|
+
const storedCheckpoint = await compressForStorage(serializedCheckpoint);
|
|
655
|
+
|
|
656
|
+
await this.db.transaction(async () => {
|
|
657
|
+
await this.db.run(
|
|
658
|
+
`INSERT OR REPLACE INTO checkpoints (thread_id, checkpoint_ns, checkpoint_id, parent_checkpoint_id, type, checkpoint, metadata)
|
|
659
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
|
660
|
+
[
|
|
661
|
+
threadId,
|
|
662
|
+
checkpointNs,
|
|
663
|
+
checkpoint.id,
|
|
664
|
+
typeof parentCheckpointId === 'string' ? parentCheckpointId : null,
|
|
665
|
+
type1,
|
|
666
|
+
storedCheckpoint,
|
|
667
|
+
serializedMetadata,
|
|
668
|
+
],
|
|
669
|
+
);
|
|
670
|
+
for (const row of messageRows) {
|
|
671
|
+
await this.db.run(
|
|
672
|
+
`INSERT OR REPLACE INTO messages (thread_id, checkpoint_ns, checkpoint_id, message_id, message_type, message_content, message, created_at)
|
|
673
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
674
|
+
row,
|
|
675
|
+
);
|
|
676
|
+
}
|
|
677
|
+
});
|
|
678
|
+
|
|
679
|
+
await this.pruneThread(threadId, checkpointNs);
|
|
680
|
+
|
|
681
|
+
return {
|
|
682
|
+
configurable: {
|
|
683
|
+
thread_id: threadId,
|
|
684
|
+
checkpoint_ns: checkpointNs,
|
|
685
|
+
checkpoint_id: checkpoint.id,
|
|
686
|
+
},
|
|
687
|
+
};
|
|
688
|
+
}
|
|
689
|
+
|
|
690
|
+
/** Normalize a message's `additional_kwargs` and serialize it exactly as the Node runtime does. */
|
|
691
|
+
private async toMessageRow(
|
|
692
|
+
message: BaseMessage,
|
|
693
|
+
threadId: string,
|
|
694
|
+
checkpointNs: string,
|
|
695
|
+
checkpointId: string,
|
|
696
|
+
): Promise<SqlParam[]> {
|
|
697
|
+
const original = message.additional_kwargs;
|
|
698
|
+
const msgFromMatrixRoom = original.msgFromMatrixRoom === true;
|
|
699
|
+
const cleaned = cleanAdditionalKwargs(
|
|
700
|
+
original,
|
|
701
|
+
msgFromMatrixRoom,
|
|
702
|
+
this.oracleName,
|
|
703
|
+
);
|
|
704
|
+
// Same shape the Node runtime writes: AI messages always carry the two
|
|
705
|
+
// reasoning keys (possibly undefined, which `_default` encodes as an
|
|
706
|
+
// explicit `{ lc: 2, type: 'undefined' }`); other message types never do.
|
|
707
|
+
const merged: CleanAdditionalKwargs = {
|
|
708
|
+
...cleaned,
|
|
709
|
+
reasoning: cleaned.reasoning ?? asOptionalString(original.reasoning),
|
|
710
|
+
reasoningDetails:
|
|
711
|
+
cleaned.reasoningDetails ??
|
|
712
|
+
asReasoningDetails(original.reasoningDetails),
|
|
713
|
+
};
|
|
714
|
+
if (message.getType() !== 'ai') {
|
|
715
|
+
delete merged.reasoning;
|
|
716
|
+
delete merged.reasoningDetails;
|
|
717
|
+
}
|
|
718
|
+
message.additional_kwargs = merged;
|
|
719
|
+
|
|
720
|
+
const serialized = await compressForStorage(
|
|
721
|
+
stringify(message, (_key, value) => _default(value)),
|
|
722
|
+
);
|
|
723
|
+
const messageId =
|
|
724
|
+
message.id ??
|
|
725
|
+
(typeof message.lc_kwargs.id === 'string'
|
|
726
|
+
? message.lc_kwargs.id
|
|
727
|
+
: undefined);
|
|
728
|
+
if (messageId === undefined) throw new Error('message is missing an id');
|
|
729
|
+
return [
|
|
730
|
+
threadId,
|
|
731
|
+
checkpointNs,
|
|
732
|
+
checkpointId,
|
|
733
|
+
messageId,
|
|
734
|
+
message.getType(),
|
|
735
|
+
typeof message.content === 'string'
|
|
736
|
+
? message.content
|
|
737
|
+
: JSON.stringify(message.content),
|
|
738
|
+
serialized,
|
|
739
|
+
merged.timestamp,
|
|
740
|
+
];
|
|
741
|
+
}
|
|
742
|
+
|
|
743
|
+
/**
|
|
744
|
+
* Drop checkpoints (and their writes) beyond the newest
|
|
745
|
+
* `maxCheckpointsPerThread` for a thread. `messages` is left intact — it is
|
|
746
|
+
* the user-visible transcript.
|
|
747
|
+
*/
|
|
748
|
+
protected async pruneThread(
|
|
749
|
+
threadId: string,
|
|
750
|
+
checkpointNs: string,
|
|
751
|
+
): Promise<void> {
|
|
752
|
+
const keep = this.maxCheckpointsPerThread;
|
|
753
|
+
if (keep <= 0) return;
|
|
754
|
+
const row = await this.db.get<{ count: number }>(
|
|
755
|
+
'SELECT COUNT(*) AS count FROM checkpoints WHERE thread_id = ? AND checkpoint_ns = ?',
|
|
756
|
+
[threadId, checkpointNs],
|
|
757
|
+
);
|
|
758
|
+
if (row === undefined || row.count <= keep + PRUNE_SLACK) return;
|
|
759
|
+
const survivors = `SELECT checkpoint_id FROM checkpoints WHERE thread_id = ? AND checkpoint_ns = ? ORDER BY checkpoint_id DESC LIMIT ?`;
|
|
760
|
+
const args = [threadId, checkpointNs, threadId, checkpointNs, keep];
|
|
761
|
+
await this.db.transaction(async () => {
|
|
762
|
+
await this.db.run(
|
|
763
|
+
`DELETE FROM writes WHERE thread_id = ? AND checkpoint_ns = ? AND checkpoint_id NOT IN (${survivors})`,
|
|
764
|
+
args,
|
|
765
|
+
);
|
|
766
|
+
await this.db.run(
|
|
767
|
+
`DELETE FROM checkpoints WHERE thread_id = ? AND checkpoint_ns = ? AND checkpoint_id NOT IN (${survivors})`,
|
|
768
|
+
args,
|
|
769
|
+
);
|
|
770
|
+
});
|
|
771
|
+
// Hand freed pages back; no-op on files whose auto_vacuum is still NONE.
|
|
772
|
+
await this.db.run('PRAGMA incremental_vacuum');
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
async putWrites(
|
|
776
|
+
config: RunnableConfig,
|
|
777
|
+
writes: PendingWrite[],
|
|
778
|
+
taskId: string,
|
|
779
|
+
): Promise<void> {
|
|
780
|
+
await this.setup();
|
|
781
|
+
if (!config.configurable) throw new Error('Empty configuration supplied.');
|
|
782
|
+
const configurable = config.configurable;
|
|
783
|
+
const givenThreadId: unknown = configurable.thread_id;
|
|
784
|
+
const checkpointId: unknown = configurable.checkpoint_id;
|
|
785
|
+
let resolvedThreadId: string;
|
|
786
|
+
if (typeof givenThreadId === 'string' && givenThreadId.length > 0) {
|
|
787
|
+
resolvedThreadId = givenThreadId;
|
|
788
|
+
} else {
|
|
789
|
+
console.error(
|
|
790
|
+
'[sqlite-saver] missing thread_id field in config.configurable',
|
|
791
|
+
configurable,
|
|
792
|
+
);
|
|
793
|
+
const found =
|
|
794
|
+
typeof checkpointId === 'string'
|
|
795
|
+
? await this.db.get<{ thread_id: string }>(
|
|
796
|
+
'SELECT thread_id FROM checkpoints WHERE checkpoint_id = ?',
|
|
797
|
+
[checkpointId],
|
|
798
|
+
)
|
|
799
|
+
: undefined;
|
|
800
|
+
if (found === undefined) {
|
|
801
|
+
throw new Error(
|
|
802
|
+
'Missing thread_id field in config.configurable. config: ' +
|
|
803
|
+
JSON.stringify(configurable),
|
|
804
|
+
);
|
|
805
|
+
}
|
|
806
|
+
resolvedThreadId = found.thread_id;
|
|
807
|
+
configurable.thread_id = found.thread_id;
|
|
808
|
+
}
|
|
809
|
+
if (typeof checkpointId !== 'string' || checkpointId.length === 0) {
|
|
810
|
+
console.error(
|
|
811
|
+
'[sqlite-saver] missing checkpoint_id field in config.configurable',
|
|
812
|
+
configurable,
|
|
813
|
+
);
|
|
814
|
+
throw new Error(
|
|
815
|
+
'Missing checkpoint_id field in config.configurable. config: ' +
|
|
816
|
+
JSON.stringify(configurable),
|
|
817
|
+
);
|
|
818
|
+
}
|
|
819
|
+
const checkpointNs: unknown = configurable.checkpoint_ns;
|
|
820
|
+
const ns = typeof checkpointNs === 'string' ? checkpointNs : '';
|
|
821
|
+
|
|
822
|
+
const rows = await Promise.all(
|
|
823
|
+
writes.map(async (write, idx): Promise<SqlParam[]> => {
|
|
824
|
+
const [type, serialized] = await this.serde.dumpsTyped(write[1]);
|
|
825
|
+
return [
|
|
826
|
+
resolvedThreadId,
|
|
827
|
+
ns,
|
|
828
|
+
checkpointId,
|
|
829
|
+
taskId,
|
|
830
|
+
idx,
|
|
831
|
+
write[0],
|
|
832
|
+
type,
|
|
833
|
+
await compressForStorage(serialized),
|
|
834
|
+
];
|
|
835
|
+
}),
|
|
836
|
+
);
|
|
837
|
+
await this.db.transaction(async () => {
|
|
838
|
+
for (const row of rows) {
|
|
839
|
+
await this.db.run(
|
|
840
|
+
`INSERT OR REPLACE INTO writes (thread_id, checkpoint_ns, checkpoint_id, task_id, idx, channel, type, value)
|
|
841
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
842
|
+
row,
|
|
843
|
+
);
|
|
844
|
+
}
|
|
845
|
+
});
|
|
846
|
+
}
|
|
847
|
+
|
|
848
|
+
async deleteThread(threadId: string): Promise<void> {
|
|
849
|
+
await this.setup();
|
|
850
|
+
await this.db.transaction(async () => {
|
|
851
|
+
await this.db.run('DELETE FROM checkpoints WHERE thread_id = ?', [
|
|
852
|
+
threadId,
|
|
853
|
+
]);
|
|
854
|
+
await this.db.run('DELETE FROM writes WHERE thread_id = ?', [threadId]);
|
|
855
|
+
await this.db.run('DELETE FROM messages WHERE thread_id = ?', [threadId]);
|
|
856
|
+
});
|
|
857
|
+
}
|
|
858
|
+
|
|
859
|
+
/** Attach the parent's `TASKS` writes as `pending_sends` on a pre-v4 checkpoint. */
|
|
860
|
+
protected async migratePendingSends(
|
|
861
|
+
checkpoint: Checkpoint,
|
|
862
|
+
threadId: string,
|
|
863
|
+
parentCheckpointId: string,
|
|
864
|
+
): Promise<void> {
|
|
865
|
+
const row = await this.db.get<{ pending_sends: string }>(
|
|
866
|
+
`SELECT json_group_array(json_object('type', ps.type, 'value', hex(ps.value))) AS pending_sends
|
|
867
|
+
FROM writes AS ps
|
|
868
|
+
WHERE ps.thread_id = ? AND ps.checkpoint_id = ? AND ps.channel = '${TASKS}'
|
|
869
|
+
ORDER BY ps.idx`,
|
|
870
|
+
[threadId, parentCheckpointId],
|
|
871
|
+
);
|
|
872
|
+
const pendingSends = JSON.parse(
|
|
873
|
+
row?.pending_sends ?? '[]',
|
|
874
|
+
) as PendingSendColumn[];
|
|
875
|
+
checkpoint.channel_values ??= {};
|
|
876
|
+
checkpoint.channel_values[TASKS] = await Promise.all(
|
|
877
|
+
pendingSends.map(async ({ type, value }) =>
|
|
878
|
+
this.serde.loadsTyped(type, await hexColumnText(value)),
|
|
879
|
+
),
|
|
880
|
+
);
|
|
881
|
+
checkpoint.channel_versions[TASKS] =
|
|
882
|
+
Object.keys(checkpoint.channel_versions).length > 0
|
|
883
|
+
? maxChannelVersion(...Object.values(checkpoint.channel_versions))
|
|
884
|
+
: this.getNextVersion(undefined);
|
|
885
|
+
}
|
|
886
|
+
}
|
|
887
|
+
|
|
888
|
+
function asOptionalString(value: unknown): string | undefined {
|
|
889
|
+
return typeof value === 'string' ? value : undefined;
|
|
890
|
+
}
|
|
891
|
+
|
|
892
|
+
function asReasoningDetails(
|
|
893
|
+
value: unknown,
|
|
894
|
+
): Array<{ type: string; text: string }> | undefined {
|
|
895
|
+
if (!Array.isArray(value)) return undefined;
|
|
896
|
+
return value.filter(
|
|
897
|
+
(d): d is { type: string; text: string } =>
|
|
898
|
+
typeof d === 'object' &&
|
|
899
|
+
d !== null &&
|
|
900
|
+
typeof d.type === 'string' &&
|
|
901
|
+
typeof d.text === 'string',
|
|
902
|
+
);
|
|
903
|
+
}
|
|
904
|
+
|
|
905
|
+
function isCheckpointWithMessages(
|
|
906
|
+
checkpoint: Checkpoint,
|
|
907
|
+
): checkpoint is CheckpointWithMessages {
|
|
908
|
+
return 'messages' in checkpoint.channel_values;
|
|
909
|
+
}
|
|
910
|
+
|
|
911
|
+
function removeMessagesFromCheckpoint(checkpoint: Checkpoint): {
|
|
912
|
+
checkpoint: Checkpoint;
|
|
913
|
+
messages?: BaseMessage[];
|
|
914
|
+
} {
|
|
915
|
+
if (!isCheckpointWithMessages(checkpoint)) return { checkpoint };
|
|
916
|
+
const copy = copyCheckpoint(checkpoint);
|
|
917
|
+
delete copy.channel_values.messages;
|
|
918
|
+
return { checkpoint: copy, messages: checkpoint.channel_values.messages };
|
|
919
|
+
}
|