@nessielabs/daemon 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -5
- package/dist/api/http.js +19 -0
- package/dist/cli.js +8 -1
- package/dist/commands/setup.js +1 -4
- package/dist/config/paths.js +1 -1
- package/dist/config/store.js +0 -36
- package/dist/localStore/daemonGraphStore.js +494 -0
- package/dist/localStore/messageComparison.js +37 -0
- package/dist/localStore/pushMapping.js +69 -0
- package/dist/localStore/rowTypes.js +1 -0
- package/dist/localStore/schema.js +70 -0
- package/dist/localStore/uuid.js +12 -0
- package/dist/parser/claudeCode.js +144 -23
- package/dist/parser/codex.js +8 -3
- package/dist/parser/types.js +7 -1
- package/dist/sync/client.js +51 -8
- package/dist/sync/exclusions.js +10 -0
- package/dist/sync/hash.js +4 -0
- package/dist/sync/loop.js +83 -53
- package/dist/sync/scan.js +219 -34
- package/dist/sync/slices.js +3 -14
- package/package.json +3 -1
- package/dist/sync/ids.js +0 -11
- package/dist/sync/map.js +0 -111
package/README.md
CHANGED
|
@@ -28,13 +28,11 @@ nessie-daemon stop
|
|
|
28
28
|
nessie-daemon status
|
|
29
29
|
```
|
|
30
30
|
|
|
31
|
-
The daemon registers selected local resources through sync preflight
|
|
31
|
+
The daemon registers selected local resources through sync preflight, pulls this device's cloud sync state, and pushes local agent traces.
|
|
32
32
|
|
|
33
|
-
Config is stored at `~/.config/nessie-daemon/config.json`; sync
|
|
33
|
+
Config is stored at `~/.config/nessie-daemon/config.json`; the local sync graph is stored at `~/.local/state/nessie-daemon/nessie.sqlite`.
|
|
34
34
|
|
|
35
|
-
Re-running `setup` rewrites the source selection
|
|
36
|
-
|
|
37
|
-
The first implementation uses file `mtime` and `size` cursors. When a JSONL file changes, the daemon reparses that file and relies on stable node IDs for idempotent pushes.
|
|
35
|
+
Re-running `setup` rewrites the source selection. The daemon reparses selected local history, reconciles it into SQLite, and only pushes rows whose local graph state has not been acknowledged by the cloud.
|
|
38
36
|
|
|
39
37
|
## systemd
|
|
40
38
|
|
package/dist/api/http.js
CHANGED
|
@@ -23,3 +23,22 @@ export async function postJson(baseUrl, path, body, options = {}) {
|
|
|
23
23
|
}
|
|
24
24
|
return (text ? JSON.parse(text) : {});
|
|
25
25
|
}
|
|
26
|
+
export async function getJson(baseUrl, path, options = {}) {
|
|
27
|
+
const fetchImpl = options.fetchImpl ?? fetch;
|
|
28
|
+
const url = new URL(path, baseUrl);
|
|
29
|
+
for (const [key, value] of Object.entries(options.query ?? {})) {
|
|
30
|
+
if (value !== undefined)
|
|
31
|
+
url.searchParams.set(key, value);
|
|
32
|
+
}
|
|
33
|
+
const response = await fetchImpl(url, {
|
|
34
|
+
method: "GET",
|
|
35
|
+
headers: {
|
|
36
|
+
...(options.token ? { authorization: `Bearer ${options.token}` } : {}),
|
|
37
|
+
},
|
|
38
|
+
});
|
|
39
|
+
const text = await response.text();
|
|
40
|
+
if (!response.ok) {
|
|
41
|
+
throw new ApiError(`HTTP status ${response.status}: ${text || response.statusText}`, response.status, text);
|
|
42
|
+
}
|
|
43
|
+
return (text ? JSON.parse(text) : {});
|
|
44
|
+
}
|
package/dist/cli.js
CHANGED
|
@@ -1,13 +1,20 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
+
import { createRequire } from "node:module";
|
|
2
3
|
import { Command } from "commander";
|
|
3
4
|
import { runDaemon } from "./commands/run.js";
|
|
4
5
|
import { runServiceCommand } from "./commands/service.js";
|
|
5
6
|
import { runSetup } from "./commands/setup.js";
|
|
7
|
+
// Read the version from package.json so the reported version can never drift
|
|
8
|
+
// from the published one. ../package.json resolves from both dist/cli.js
|
|
9
|
+
// (installed) and src/cli.ts (tsx dev path), since both sit one level under
|
|
10
|
+
// the package root, and npm always includes package.json in the tarball.
|
|
11
|
+
const require = createRequire(import.meta.url);
|
|
12
|
+
const { version } = require("../package.json");
|
|
6
13
|
const program = new Command();
|
|
7
14
|
program
|
|
8
15
|
.name("nessie-daemon")
|
|
9
16
|
.description("Headless Nessie ingestion daemon for Linux agent machines.")
|
|
10
|
-
.version(
|
|
17
|
+
.version(version);
|
|
11
18
|
program
|
|
12
19
|
.command("setup")
|
|
13
20
|
.description("Authenticate, select local sources, and optionally install the user systemd service.")
|
package/dist/commands/setup.js
CHANGED
|
@@ -2,7 +2,7 @@ import { checkbox, confirm } from "@inquirer/prompts";
|
|
|
2
2
|
import { hostname } from "node:os";
|
|
3
3
|
import { runDeviceAuth } from "../auth/deviceFlow.js";
|
|
4
4
|
import { getDaemonPaths } from "../config/paths.js";
|
|
5
|
-
import { createConfig,
|
|
5
|
+
import { createConfig, writeConfig } from "../config/store.js";
|
|
6
6
|
import { installUserService } from "../service/systemd.js";
|
|
7
7
|
import { detectSources } from "../sources/detect.js";
|
|
8
8
|
export async function runSetup(options, depsOverride = {}) {
|
|
@@ -38,7 +38,6 @@ export async function runSetup(options, depsOverride = {}) {
|
|
|
38
38
|
selectedSources,
|
|
39
39
|
});
|
|
40
40
|
await deps.writeConfig(config, deps.paths);
|
|
41
|
-
await deps.writeState(createInitialState(), deps.paths);
|
|
42
41
|
console.log(`Saved config to ${deps.paths.configFile}`);
|
|
43
42
|
if (await confirm({ message: "Install and start the user systemd service?", default: true })) {
|
|
44
43
|
await installUserService(deps.paths);
|
|
@@ -59,7 +58,6 @@ async function runAPIKeySetup(input) {
|
|
|
59
58
|
selectedSources: detectedSources,
|
|
60
59
|
});
|
|
61
60
|
await input.deps.writeConfig(config, input.deps.paths);
|
|
62
|
-
await input.deps.writeState(createInitialState(), input.deps.paths);
|
|
63
61
|
console.log("Configured Nessie daemon with API key auth.");
|
|
64
62
|
console.log(`Selected ${detectedSources.length} source(s):`);
|
|
65
63
|
for (const source of detectedSources) {
|
|
@@ -72,7 +70,6 @@ function resolveSetupDeps(overrides) {
|
|
|
72
70
|
return {
|
|
73
71
|
detectSources: overrides.detectSources ?? detectSources,
|
|
74
72
|
writeConfig: overrides.writeConfig ?? writeConfig,
|
|
75
|
-
writeState: overrides.writeState ?? writeState,
|
|
76
73
|
paths: overrides.paths ?? getDaemonPaths(),
|
|
77
74
|
};
|
|
78
75
|
}
|
package/dist/config/paths.js
CHANGED
|
@@ -10,7 +10,7 @@ export function getDaemonPaths(env = process.env, home = homedir()) {
|
|
|
10
10
|
configDir,
|
|
11
11
|
stateDir,
|
|
12
12
|
configFile: join(configDir, "config.json"),
|
|
13
|
-
|
|
13
|
+
databaseFile: join(stateDir, "nessie.sqlite"),
|
|
14
14
|
pidFile: join(stateDir, "daemon.pid"),
|
|
15
15
|
logFile: join(stateDir, "daemon.log"),
|
|
16
16
|
systemdUserDir,
|
package/dist/config/store.js
CHANGED
|
@@ -2,7 +2,6 @@ import { randomUUID } from "node:crypto";
|
|
|
2
2
|
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
3
3
|
import { hostname } from "node:os";
|
|
4
4
|
import { dirname } from "node:path";
|
|
5
|
-
import { PARSER_VERSION } from "../parser/types.js";
|
|
6
5
|
import { getDaemonPaths } from "./paths.js";
|
|
7
6
|
export function createConfig(input) {
|
|
8
7
|
const deviceId = input.deviceId ?? randomUUID();
|
|
@@ -19,15 +18,6 @@ export function createConfig(input) {
|
|
|
19
18
|
})),
|
|
20
19
|
};
|
|
21
20
|
}
|
|
22
|
-
export function createInitialState() {
|
|
23
|
-
return {
|
|
24
|
-
files: {},
|
|
25
|
-
pushedSourceRootIds: [],
|
|
26
|
-
pushedSessions: {},
|
|
27
|
-
parserVersion: PARSER_VERSION,
|
|
28
|
-
lastSyncAt: null,
|
|
29
|
-
};
|
|
30
|
-
}
|
|
31
21
|
export async function readConfig(paths = getDaemonPaths()) {
|
|
32
22
|
return JSON.parse(await readFile(paths.configFile, "utf8"));
|
|
33
23
|
}
|
|
@@ -35,29 +25,3 @@ export async function writeConfig(config, paths = getDaemonPaths()) {
|
|
|
35
25
|
await mkdir(dirname(paths.configFile), { recursive: true });
|
|
36
26
|
await writeFile(paths.configFile, `${JSON.stringify(config, null, 2)}\n`, { mode: 0o600 });
|
|
37
27
|
}
|
|
38
|
-
export async function readState(paths = getDaemonPaths()) {
|
|
39
|
-
try {
|
|
40
|
-
const parsed = JSON.parse(await readFile(paths.stateFile, "utf8"));
|
|
41
|
-
return {
|
|
42
|
-
files: parsed.files ?? {},
|
|
43
|
-
pushedSourceRootIds: parsed.pushedSourceRootIds ?? [],
|
|
44
|
-
pushedSessions: parsed.pushedSessions ?? {},
|
|
45
|
-
// State written before parser versioning existed predates every
|
|
46
|
-
// versioned parser change, so it always needs the backfill pass.
|
|
47
|
-
parserVersion: parsed.parserVersion ?? 0,
|
|
48
|
-
lastSyncAt: parsed.lastSyncAt ?? null,
|
|
49
|
-
};
|
|
50
|
-
}
|
|
51
|
-
catch (error) {
|
|
52
|
-
if (isMissingFileError(error))
|
|
53
|
-
return createInitialState();
|
|
54
|
-
throw error;
|
|
55
|
-
}
|
|
56
|
-
}
|
|
57
|
-
export async function writeState(state, paths = getDaemonPaths()) {
|
|
58
|
-
await mkdir(dirname(paths.stateFile), { recursive: true });
|
|
59
|
-
await writeFile(paths.stateFile, `${JSON.stringify(state, null, 2)}\n`, { mode: 0o600 });
|
|
60
|
-
}
|
|
61
|
-
function isMissingFileError(error) {
|
|
62
|
-
return typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT";
|
|
63
|
-
}
|
|
@@ -0,0 +1,494 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { mkdirSync } from "node:fs";
|
|
3
|
+
import { dirname } from "node:path";
|
|
4
|
+
import Database from "better-sqlite3";
|
|
5
|
+
import { redactSecrets } from "../redact/secrets.js";
|
|
6
|
+
import { contentHash } from "../sync/hash.js";
|
|
7
|
+
import { splitContent } from "../sync/slices.js";
|
|
8
|
+
import { messageMatches, messageMetadataChanged, messageNodeName, messageTimestamp, } from "./messageComparison.js";
|
|
9
|
+
import { mapEdgePushItem, mapNodePushItem } from "./pushMapping.js";
|
|
10
|
+
import { createDaemonGraphSchema } from "./schema.js";
|
|
11
|
+
import { uuidFromBytes, uuidToBytes } from "./uuid.js";
|
|
12
|
+
const NODE_CANDIDATE_LIMIT = 500;
|
|
13
|
+
export class DaemonGraphStore {
|
|
14
|
+
db;
|
|
15
|
+
lastTimestampMs = 0;
|
|
16
|
+
constructor(db) {
|
|
17
|
+
this.db = db;
|
|
18
|
+
createDaemonGraphSchema(db);
|
|
19
|
+
}
|
|
20
|
+
close() {
|
|
21
|
+
this.db.close();
|
|
22
|
+
}
|
|
23
|
+
persistSourceSessions(source, sessions, options) {
|
|
24
|
+
const persist = this.db.transaction(() => {
|
|
25
|
+
this.ensureIntegrationRoot(source);
|
|
26
|
+
for (const session of sessions) {
|
|
27
|
+
this.persistSession(source, session, options);
|
|
28
|
+
}
|
|
29
|
+
});
|
|
30
|
+
persist();
|
|
31
|
+
}
|
|
32
|
+
conversationsNeedingFetch(source, conversations, parserVersion) {
|
|
33
|
+
const kind = source.kind === "codex" ? "codex_chat" : "claude_code_chat";
|
|
34
|
+
const select = this.db.prepare(`
|
|
35
|
+
SELECT parserVersion, originalUpdatedAt
|
|
36
|
+
FROM node
|
|
37
|
+
WHERE kind = ? AND sourceId = ? AND deletedAt IS NULL
|
|
38
|
+
ORDER BY createdAt ASC
|
|
39
|
+
LIMIT 1
|
|
40
|
+
`);
|
|
41
|
+
this.ensureIntegrationRoot(source);
|
|
42
|
+
return conversations.filter((conversation) => {
|
|
43
|
+
const row = select.get(kind, conversation.sourceId);
|
|
44
|
+
if (!row)
|
|
45
|
+
return true;
|
|
46
|
+
const remoteUpdatedAt = conversation.lastMessageTime ?? conversation.updatedAt;
|
|
47
|
+
const isUpdated = remoteUpdatedAt
|
|
48
|
+
? Date.parse(remoteUpdatedAt) - Date.parse(row.originalUpdatedAt) > 5000
|
|
49
|
+
: false;
|
|
50
|
+
if (row.parserVersion !== null && row.parserVersion < parserVersion)
|
|
51
|
+
return true;
|
|
52
|
+
return isUpdated;
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
pendingPushItems() {
|
|
56
|
+
const nodes = this.dirtyNodes().map((node) => this.mapNodePushItem(node));
|
|
57
|
+
const edges = this.dirtyEdges().map((edge) => this.mapEdgePushItem(edge));
|
|
58
|
+
return { nodes, edges };
|
|
59
|
+
}
|
|
60
|
+
nextPushBatch(options) {
|
|
61
|
+
const nodeRows = this.dirtyNodes(NODE_CANDIDATE_LIMIT);
|
|
62
|
+
const nodes = [];
|
|
63
|
+
const pushedNodeVersions = {};
|
|
64
|
+
let sliceCount = 0;
|
|
65
|
+
for (const row of nodeRows) {
|
|
66
|
+
const node = this.mapNodePushItem(row);
|
|
67
|
+
const nodeSliceCount = node.slices.length;
|
|
68
|
+
if (nodes.length > 0 && sliceCount + nodeSliceCount > options.targetSliceCount)
|
|
69
|
+
break;
|
|
70
|
+
nodes.push(node);
|
|
71
|
+
pushedNodeVersions[node.node.id] = row.updatedAt;
|
|
72
|
+
sliceCount += nodeSliceCount;
|
|
73
|
+
}
|
|
74
|
+
if (nodes.length > 0)
|
|
75
|
+
return { nodes, edges: [], pushedNodeVersions, pushedEdgeVersions: {} };
|
|
76
|
+
const edgeRows = this.dirtyEdges(options.edgeLimit);
|
|
77
|
+
const pushedEdgeVersions = {};
|
|
78
|
+
const edges = edgeRows.map((edge) => {
|
|
79
|
+
const item = this.mapEdgePushItem(edge);
|
|
80
|
+
pushedEdgeVersions[item.edge.id] = edge.updatedAt;
|
|
81
|
+
return item;
|
|
82
|
+
});
|
|
83
|
+
return edges.length > 0 ? { nodes: [], edges, pushedNodeVersions: {}, pushedEdgeVersions } : null;
|
|
84
|
+
}
|
|
85
|
+
applyPushAcknowledgements(batch, response) {
|
|
86
|
+
const apply = this.db.transaction(() => {
|
|
87
|
+
const updateNode = this.db.prepare(`
|
|
88
|
+
UPDATE node SET lastPushedAt = ?, cloudUpdatedAt = ?
|
|
89
|
+
WHERE id = ?
|
|
90
|
+
`);
|
|
91
|
+
for (const ack of response.nodes) {
|
|
92
|
+
const pushedUpdatedAt = batch.pushedNodeVersions[ack.id];
|
|
93
|
+
if (!pushedUpdatedAt)
|
|
94
|
+
continue;
|
|
95
|
+
updateNode.run(pushedUpdatedAt, ack.updatedAt, uuidToBytes(ack.id));
|
|
96
|
+
}
|
|
97
|
+
const updateEdge = this.db.prepare(`
|
|
98
|
+
UPDATE node_edge SET lastPushedAt = ?, cloudUpdatedAt = ?
|
|
99
|
+
WHERE id = ?
|
|
100
|
+
`);
|
|
101
|
+
for (const ack of response.edges) {
|
|
102
|
+
const pushedUpdatedAt = batch.pushedEdgeVersions[ack.id];
|
|
103
|
+
if (!pushedUpdatedAt)
|
|
104
|
+
continue;
|
|
105
|
+
updateEdge.run(pushedUpdatedAt, ack.updatedAt, uuidToBytes(ack.id));
|
|
106
|
+
}
|
|
107
|
+
});
|
|
108
|
+
apply();
|
|
109
|
+
}
|
|
110
|
+
applyCloudPull(response) {
|
|
111
|
+
const apply = this.db.transaction(() => {
|
|
112
|
+
let skippedEdges = 0;
|
|
113
|
+
for (const bundle of response.nodes) {
|
|
114
|
+
this.applyCloudPullNodeBundle(bundle);
|
|
115
|
+
}
|
|
116
|
+
for (const edge of response.edges) {
|
|
117
|
+
if (!this.applyCloudPullEdge(edge))
|
|
118
|
+
skippedEdges += 1;
|
|
119
|
+
}
|
|
120
|
+
return { skippedEdges };
|
|
121
|
+
});
|
|
122
|
+
return apply();
|
|
123
|
+
}
|
|
124
|
+
getAppSetting(key) {
|
|
125
|
+
const row = this.db.prepare("SELECT value FROM app_setting WHERE key = ?").get(key);
|
|
126
|
+
return row?.value ?? null;
|
|
127
|
+
}
|
|
128
|
+
getSyncCursor(key) {
|
|
129
|
+
const value = this.getAppSetting(key);
|
|
130
|
+
if (!value)
|
|
131
|
+
return null;
|
|
132
|
+
try {
|
|
133
|
+
const parsed = JSON.parse(value);
|
|
134
|
+
return typeof parsed.updatedAt === "string" && typeof parsed.id === "string"
|
|
135
|
+
? { updatedAt: parsed.updatedAt, id: parsed.id }
|
|
136
|
+
: null;
|
|
137
|
+
}
|
|
138
|
+
catch {
|
|
139
|
+
return null;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
setAppSetting(key, value) {
|
|
143
|
+
this.db.prepare(`
|
|
144
|
+
INSERT INTO app_setting (key, value) VALUES (?, ?)
|
|
145
|
+
ON CONFLICT(key) DO UPDATE SET value = excluded.value
|
|
146
|
+
`).run(key, value);
|
|
147
|
+
}
|
|
148
|
+
setSyncCursor(key, cursor) {
|
|
149
|
+
this.setAppSetting(key, JSON.stringify(cursor));
|
|
150
|
+
}
|
|
151
|
+
ensureIntegrationRoot(source) {
|
|
152
|
+
const now = this.nextTimestamp();
|
|
153
|
+
const id = uuidToBytes(source.localId);
|
|
154
|
+
const existing = this.nodeById(id);
|
|
155
|
+
const name = redactSecrets(source.displayName);
|
|
156
|
+
if (!existing) {
|
|
157
|
+
this.insertNode({
|
|
158
|
+
id,
|
|
159
|
+
integrationId: id,
|
|
160
|
+
sourceId: source.basePath,
|
|
161
|
+
name,
|
|
162
|
+
kind: "integration_root",
|
|
163
|
+
originalCreatedAt: now,
|
|
164
|
+
originalUpdatedAt: now,
|
|
165
|
+
now,
|
|
166
|
+
});
|
|
167
|
+
return id;
|
|
168
|
+
}
|
|
169
|
+
if (existing.integrationId?.equals(id) !== true ||
|
|
170
|
+
existing.sourceId !== source.basePath ||
|
|
171
|
+
existing.name !== name ||
|
|
172
|
+
existing.kind !== "integration_root" ||
|
|
173
|
+
existing.deletedAt !== null) {
|
|
174
|
+
this.db.prepare(`
|
|
175
|
+
UPDATE node
|
|
176
|
+
SET integrationId = ?, sourceId = ?, name = ?, kind = 'integration_root',
|
|
177
|
+
updatedAt = ?, originalUpdatedAt = ?, deletedAt = NULL
|
|
178
|
+
WHERE id = ?
|
|
179
|
+
`).run(id, source.basePath, name, now, now, id);
|
|
180
|
+
}
|
|
181
|
+
return id;
|
|
182
|
+
}
|
|
183
|
+
persistSession(source, session, options) {
|
|
184
|
+
const now = this.nextTimestamp();
|
|
185
|
+
const transcriptKind = session.sourceKind === "codex" ? "codex_chat" : "claude_code_chat";
|
|
186
|
+
const messageKind = session.sourceKind === "codex" ? "codex_chat_message" : "claude_code_chat_message";
|
|
187
|
+
const integrationId = uuidToBytes(source.localId);
|
|
188
|
+
const repoInfo = session.cwd ? options.resolveRepo?.(session.cwd) ?? null : null;
|
|
189
|
+
const transcript = this.upsertTranscript({
|
|
190
|
+
integrationId,
|
|
191
|
+
sourceId: session.sourceId,
|
|
192
|
+
name: redactSecrets(session.title),
|
|
193
|
+
kind: transcriptKind,
|
|
194
|
+
workspacePath: session.cwd,
|
|
195
|
+
repoRemoteUrl: repoInfo?.remoteUrl ?? null,
|
|
196
|
+
repoKey: repoInfo?.repoKey ?? null,
|
|
197
|
+
parserVersion: options.parserVersion,
|
|
198
|
+
originalCreatedAt: session.createdAt,
|
|
199
|
+
originalUpdatedAt: session.updatedAt,
|
|
200
|
+
now,
|
|
201
|
+
});
|
|
202
|
+
this.ensureSourceEdge(integrationId, transcript.id, now);
|
|
203
|
+
const existingMessages = this.existingMessages(transcript.id);
|
|
204
|
+
const pushableMessages = session.messages.filter((message) => message.role !== "system");
|
|
205
|
+
const keptIds = new Set();
|
|
206
|
+
for (const [index, message] of pushableMessages.entries()) {
|
|
207
|
+
const existing = existingMessages[index];
|
|
208
|
+
if (existing && messageMatches(existing, message, messageKind)) {
|
|
209
|
+
const timestamp = messageTimestamp(message, session);
|
|
210
|
+
if (messageMetadataChanged(existing.node, message, integrationId, messageKind, timestamp)) {
|
|
211
|
+
this.db.prepare(`
|
|
212
|
+
UPDATE node
|
|
213
|
+
SET integrationId = ?, name = ?, kind = ?, originalCreatedAt = ?,
|
|
214
|
+
originalUpdatedAt = ?, updatedAt = ?, deletedAt = NULL
|
|
215
|
+
WHERE id = ?
|
|
216
|
+
`).run(integrationId, messageNodeName(message), messageKind, timestamp, timestamp, now, existing.node.id);
|
|
217
|
+
}
|
|
218
|
+
this.ensureSourceEdge(integrationId, existing.node.id, now);
|
|
219
|
+
this.ensureContainsEdge(transcript.id, existing.node.id, index, now);
|
|
220
|
+
keptIds.add(uuidFromBytes(existing.node.id));
|
|
221
|
+
continue;
|
|
222
|
+
}
|
|
223
|
+
const nodeId = uuidToBytes(randomUUID());
|
|
224
|
+
const content = redactSecrets(message.content);
|
|
225
|
+
const timestamp = messageTimestamp(message, session);
|
|
226
|
+
this.insertNode({
|
|
227
|
+
id: nodeId,
|
|
228
|
+
integrationId,
|
|
229
|
+
sourceId: null,
|
|
230
|
+
name: messageNodeName(message),
|
|
231
|
+
kind: messageKind,
|
|
232
|
+
originalCreatedAt: timestamp,
|
|
233
|
+
originalUpdatedAt: timestamp,
|
|
234
|
+
now,
|
|
235
|
+
});
|
|
236
|
+
this.db.prepare(`
|
|
237
|
+
INSERT INTO ai_chat_message (nodeId, role, author, toolCallId, toolName, toolFriendlyName, toolStatus)
|
|
238
|
+
VALUES (?, ?, NULL, NULL, NULL, NULL, NULL)
|
|
239
|
+
`).run(nodeId, message.role);
|
|
240
|
+
this.replaceContentSlices(nodeId, content);
|
|
241
|
+
this.ensureContainsEdge(transcript.id, nodeId, index, now);
|
|
242
|
+
this.ensureSourceEdge(integrationId, nodeId, now);
|
|
243
|
+
keptIds.add(uuidFromBytes(nodeId));
|
|
244
|
+
}
|
|
245
|
+
for (const existing of existingMessages) {
|
|
246
|
+
if (!keptIds.has(uuidFromBytes(existing.node.id)))
|
|
247
|
+
this.softDeleteNode(existing.node.id, now);
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
upsertTranscript(input) {
|
|
251
|
+
const existing = this.db.prepare(`
|
|
252
|
+
SELECT * FROM node
|
|
253
|
+
WHERE kind = ? AND sourceId = ?
|
|
254
|
+
ORDER BY createdAt ASC
|
|
255
|
+
LIMIT 1
|
|
256
|
+
`).get(input.kind, input.sourceId);
|
|
257
|
+
if (!existing) {
|
|
258
|
+
const id = uuidToBytes(randomUUID());
|
|
259
|
+
this.insertNode({ id, ...input });
|
|
260
|
+
return this.nodeById(id);
|
|
261
|
+
}
|
|
262
|
+
if (existing.integrationId?.equals(input.integrationId) !== true ||
|
|
263
|
+
existing.name !== input.name ||
|
|
264
|
+
existing.workspacePath !== input.workspacePath ||
|
|
265
|
+
existing.repoRemoteUrl !== input.repoRemoteUrl ||
|
|
266
|
+
existing.repoKey !== input.repoKey ||
|
|
267
|
+
existing.parserVersion !== input.parserVersion ||
|
|
268
|
+
existing.originalCreatedAt !== input.originalCreatedAt ||
|
|
269
|
+
existing.originalUpdatedAt !== input.originalUpdatedAt ||
|
|
270
|
+
existing.deletedAt !== null) {
|
|
271
|
+
this.db.prepare(`
|
|
272
|
+
UPDATE node
|
|
273
|
+
SET integrationId = ?, name = ?, workspacePath = ?, repoRemoteUrl = ?,
|
|
274
|
+
repoKey = ?, parserVersion = ?, originalCreatedAt = ?, originalUpdatedAt = ?,
|
|
275
|
+
updatedAt = ?, deletedAt = NULL
|
|
276
|
+
WHERE id = ?
|
|
277
|
+
`).run(input.integrationId, input.name, input.workspacePath, input.repoRemoteUrl, input.repoKey, input.parserVersion, input.originalCreatedAt, input.originalUpdatedAt, input.now, existing.id);
|
|
278
|
+
return this.nodeById(existing.id);
|
|
279
|
+
}
|
|
280
|
+
return existing;
|
|
281
|
+
}
|
|
282
|
+
insertNode(input) {
|
|
283
|
+
this.db.prepare(`
|
|
284
|
+
INSERT INTO node (
|
|
285
|
+
id, integrationId, sourceId, name, kind, pinned, workspacePath,
|
|
286
|
+
repoRemoteUrl, repoKey, createdAt, updatedAt, parserVersion,
|
|
287
|
+
originalCreatedAt, originalUpdatedAt
|
|
288
|
+
)
|
|
289
|
+
VALUES (?, ?, ?, ?, ?, 0, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
290
|
+
`).run(input.id, input.integrationId, input.sourceId, input.name, input.kind, input.workspacePath ?? null, input.repoRemoteUrl ?? null, input.repoKey ?? null, input.now, input.now, input.parserVersion ?? null, input.originalCreatedAt, input.originalUpdatedAt);
|
|
291
|
+
}
|
|
292
|
+
ensureContainsEdge(fromNodeId, toNodeId, sortIndex, now) {
|
|
293
|
+
this.ensureEdge(fromNodeId, toNodeId, "contains", sortIndex, now);
|
|
294
|
+
}
|
|
295
|
+
ensureSourceEdge(fromNodeId, toNodeId, now) {
|
|
296
|
+
this.ensureEdge(fromNodeId, toNodeId, "source_of", 0, now);
|
|
297
|
+
}
|
|
298
|
+
ensureEdge(fromNodeId, toNodeId, type, sortIndex, now) {
|
|
299
|
+
const existing = this.db.prepare(`
|
|
300
|
+
SELECT * FROM node_edge
|
|
301
|
+
WHERE fromNodeId = ? AND toNodeId = ? AND type = ?
|
|
302
|
+
LIMIT 1
|
|
303
|
+
`).get(fromNodeId, toNodeId, type);
|
|
304
|
+
if (!existing) {
|
|
305
|
+
this.db.prepare(`
|
|
306
|
+
INSERT INTO node_edge (id, fromNodeId, toNodeId, type, sortIndex, createdAt, updatedAt)
|
|
307
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
308
|
+
`).run(uuidToBytes(randomUUID()), fromNodeId, toNodeId, type, sortIndex, now, now);
|
|
309
|
+
return;
|
|
310
|
+
}
|
|
311
|
+
if (existing.sortIndex !== sortIndex || existing.deletedAt !== null) {
|
|
312
|
+
this.db.prepare(`
|
|
313
|
+
UPDATE node_edge
|
|
314
|
+
SET sortIndex = ?, updatedAt = ?, deletedAt = NULL
|
|
315
|
+
WHERE id = ?
|
|
316
|
+
`).run(sortIndex, now, existing.id);
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
replaceContentSlices(nodeId, content) {
|
|
320
|
+
this.db.prepare("DELETE FROM node_content_slice WHERE nodeId = ?").run(nodeId);
|
|
321
|
+
const insert = this.db.prepare(`
|
|
322
|
+
INSERT INTO node_content_slice (id, nodeId, "index", content, contentHash)
|
|
323
|
+
VALUES (?, ?, ?, ?, ?)
|
|
324
|
+
`);
|
|
325
|
+
for (const [index, piece] of splitContent(content).entries()) {
|
|
326
|
+
insert.run(uuidToBytes(randomUUID()), nodeId, index, piece, contentHash(piece));
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
applyCloudPullNodeBundle(bundle) {
|
|
330
|
+
const nodeId = uuidToBytes(bundle.node.id);
|
|
331
|
+
const existing = this.nodeById(nodeId);
|
|
332
|
+
if (existing?.cloudUpdatedAt === bundle.node.updatedAt)
|
|
333
|
+
return;
|
|
334
|
+
const now = this.nextTimestamp();
|
|
335
|
+
this.db.prepare(`
|
|
336
|
+
INSERT INTO node (
|
|
337
|
+
id, integrationId, sourceId, name, kind, emoji, pinned, label, labelSliceCount,
|
|
338
|
+
workspacePath, repoRemoteUrl, repoKey, createdAt, updatedAt, lastPushedAt,
|
|
339
|
+
cloudUpdatedAt, parserVersion, deletedAt, originalCreatedAt, originalUpdatedAt
|
|
340
|
+
)
|
|
341
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
342
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
343
|
+
integrationId = excluded.integrationId,
|
|
344
|
+
sourceId = excluded.sourceId,
|
|
345
|
+
name = excluded.name,
|
|
346
|
+
kind = excluded.kind,
|
|
347
|
+
emoji = excluded.emoji,
|
|
348
|
+
pinned = excluded.pinned,
|
|
349
|
+
label = excluded.label,
|
|
350
|
+
labelSliceCount = excluded.labelSliceCount,
|
|
351
|
+
workspacePath = excluded.workspacePath,
|
|
352
|
+
repoRemoteUrl = excluded.repoRemoteUrl,
|
|
353
|
+
repoKey = excluded.repoKey,
|
|
354
|
+
updatedAt = excluded.updatedAt,
|
|
355
|
+
lastPushedAt = excluded.lastPushedAt,
|
|
356
|
+
cloudUpdatedAt = excluded.cloudUpdatedAt,
|
|
357
|
+
deletedAt = excluded.deletedAt,
|
|
358
|
+
originalCreatedAt = excluded.originalCreatedAt,
|
|
359
|
+
originalUpdatedAt = excluded.originalUpdatedAt
|
|
360
|
+
`).run(nodeId, bundle.node.integrationId ? uuidToBytes(bundle.node.integrationId) : null, bundle.node.sourceId ?? null, bundle.node.name, bundle.node.kind, bundle.node.emoji ?? null, bundle.node.pinned === true ? 1 : 0, bundle.node.label ?? null, bundle.node.labelSliceCount ?? null, bundle.node.workspacePath ?? null, bundle.node.repoRemoteUrl ?? null, bundle.node.repoKey ?? null, existing?.createdAt ?? now, now, now, bundle.node.updatedAt, existing?.parserVersion ?? null, bundle.node.deletedAt ?? null, bundle.node.originalCreatedAt, bundle.node.originalUpdatedAt);
|
|
361
|
+
this.replacePulledContentSlices(nodeId, bundle.slices);
|
|
362
|
+
this.replacePulledAiChatMessage(nodeId, bundle.aiChatMessage ?? null);
|
|
363
|
+
}
|
|
364
|
+
applyCloudPullEdge(edge) {
|
|
365
|
+
const id = uuidToBytes(edge.id);
|
|
366
|
+
const fromNodeId = uuidToBytes(edge.fromNodeId);
|
|
367
|
+
const toNodeId = uuidToBytes(edge.toNodeId);
|
|
368
|
+
if (!this.nodeById(fromNodeId) || !this.nodeById(toNodeId))
|
|
369
|
+
return false;
|
|
370
|
+
const existing = this.db.prepare("SELECT createdAt, cloudUpdatedAt FROM node_edge WHERE id = ?")
|
|
371
|
+
.get(id);
|
|
372
|
+
if (existing?.cloudUpdatedAt === edge.updatedAt)
|
|
373
|
+
return true;
|
|
374
|
+
const now = this.nextTimestamp();
|
|
375
|
+
this.db.prepare(`
|
|
376
|
+
INSERT INTO node_edge (
|
|
377
|
+
id, fromNodeId, toNodeId, type, sortIndex, createdAt, updatedAt,
|
|
378
|
+
lastPushedAt, cloudUpdatedAt, deletedAt
|
|
379
|
+
)
|
|
380
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
381
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
382
|
+
fromNodeId = excluded.fromNodeId,
|
|
383
|
+
toNodeId = excluded.toNodeId,
|
|
384
|
+
type = excluded.type,
|
|
385
|
+
sortIndex = excluded.sortIndex,
|
|
386
|
+
updatedAt = excluded.updatedAt,
|
|
387
|
+
lastPushedAt = excluded.lastPushedAt,
|
|
388
|
+
cloudUpdatedAt = excluded.cloudUpdatedAt,
|
|
389
|
+
deletedAt = excluded.deletedAt
|
|
390
|
+
`).run(id, fromNodeId, toNodeId, edge.type, edge.sortIndex ?? null, existing?.createdAt ?? now, now, now, edge.updatedAt, edge.deletedAt ?? null);
|
|
391
|
+
return true;
|
|
392
|
+
}
|
|
393
|
+
replacePulledContentSlices(nodeId, slices) {
|
|
394
|
+
this.db.prepare("DELETE FROM node_content_slice WHERE nodeId = ?").run(nodeId);
|
|
395
|
+
const insert = this.db.prepare(`
|
|
396
|
+
INSERT INTO node_content_slice (id, nodeId, "index", content, contentHash)
|
|
397
|
+
VALUES (?, ?, ?, ?, ?)
|
|
398
|
+
`);
|
|
399
|
+
for (const slice of slices) {
|
|
400
|
+
insert.run(uuidToBytes(slice.id), nodeId, slice.index, slice.content, slice.contentHash);
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
replacePulledAiChatMessage(nodeId, message) {
|
|
404
|
+
this.db.prepare("DELETE FROM ai_chat_message WHERE nodeId = ?").run(nodeId);
|
|
405
|
+
if (!message)
|
|
406
|
+
return;
|
|
407
|
+
this.db.prepare(`
|
|
408
|
+
INSERT INTO ai_chat_message (nodeId, role, author, toolCallId, toolName, toolFriendlyName, toolStatus)
|
|
409
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
410
|
+
`).run(nodeId, message.role, message.author ?? null, message.toolCallId ?? null, message.toolName ?? null, message.toolFriendlyName ?? null, message.toolStatus ?? null);
|
|
411
|
+
}
|
|
412
|
+
softDeleteNode(id, now) {
|
|
413
|
+
this.db.prepare("UPDATE node SET deletedAt = ?, updatedAt = ? WHERE id = ? AND deletedAt IS NULL")
|
|
414
|
+
.run(now, now, id);
|
|
415
|
+
this.db.prepare(`
|
|
416
|
+
UPDATE node_edge
|
|
417
|
+
SET deletedAt = ?, updatedAt = ?
|
|
418
|
+
WHERE (fromNodeId = ? OR toNodeId = ?) AND deletedAt IS NULL
|
|
419
|
+
`).run(now, now, id, id);
|
|
420
|
+
}
|
|
421
|
+
existingMessages(transcriptId) {
|
|
422
|
+
const rows = this.db.prepare(`
|
|
423
|
+
SELECT n.*, ai.role, ai.author, ai.toolCallId, ai.toolName, ai.toolStatus
|
|
424
|
+
FROM node_edge e
|
|
425
|
+
JOIN node n ON n.id = e.toNodeId
|
|
426
|
+
LEFT JOIN ai_chat_message ai ON ai.nodeId = n.id
|
|
427
|
+
WHERE e.fromNodeId = ? AND e.type = 'contains'
|
|
428
|
+
AND e.deletedAt IS NULL AND n.deletedAt IS NULL
|
|
429
|
+
ORDER BY e.sortIndex ASC, n.createdAt ASC
|
|
430
|
+
`).all(transcriptId);
|
|
431
|
+
return rows.map((row) => ({
|
|
432
|
+
node: row,
|
|
433
|
+
role: row.role,
|
|
434
|
+
author: row.author,
|
|
435
|
+
toolCallId: row.toolCallId,
|
|
436
|
+
toolName: row.toolName,
|
|
437
|
+
toolStatus: row.toolStatus,
|
|
438
|
+
contentHashes: this.contentHashes(row.id),
|
|
439
|
+
}));
|
|
440
|
+
}
|
|
441
|
+
contentHashes(nodeId) {
|
|
442
|
+
const rows = this.db.prepare(`
|
|
443
|
+
SELECT contentHash
|
|
444
|
+
FROM node_content_slice
|
|
445
|
+
WHERE nodeId = ?
|
|
446
|
+
ORDER BY "index" ASC
|
|
447
|
+
`).all(nodeId);
|
|
448
|
+
return rows.map((row) => row.contentHash);
|
|
449
|
+
}
|
|
450
|
+
dirtyNodes(limit) {
|
|
451
|
+
return this.db.prepare(`
|
|
452
|
+
SELECT *
|
|
453
|
+
FROM node
|
|
454
|
+
WHERE lastPushedAt IS NULL OR updatedAt > lastPushedAt
|
|
455
|
+
ORDER BY
|
|
456
|
+
CASE kind
|
|
457
|
+
WHEN 'integration_root' THEN 0
|
|
458
|
+
WHEN 'codex_chat' THEN 1
|
|
459
|
+
WHEN 'claude_code_chat' THEN 1
|
|
460
|
+
ELSE 2
|
|
461
|
+
END ASC,
|
|
462
|
+
updatedAt ASC,
|
|
463
|
+
hex(id) ASC
|
|
464
|
+
${limit === undefined ? "" : "LIMIT ?"}
|
|
465
|
+
`).all(...(limit === undefined ? [] : [limit]));
|
|
466
|
+
}
|
|
467
|
+
dirtyEdges(limit) {
|
|
468
|
+
return this.db.prepare(`
|
|
469
|
+
SELECT *
|
|
470
|
+
FROM node_edge
|
|
471
|
+
WHERE lastPushedAt IS NULL OR updatedAt > lastPushedAt
|
|
472
|
+
ORDER BY updatedAt ASC, hex(id) ASC
|
|
473
|
+
${limit === undefined ? "" : "LIMIT ?"}
|
|
474
|
+
`).all(...(limit === undefined ? [] : [limit]));
|
|
475
|
+
}
|
|
476
|
+
mapNodePushItem(row) {
|
|
477
|
+
return mapNodePushItem(this.db, row);
|
|
478
|
+
}
|
|
479
|
+
mapEdgePushItem(row) {
|
|
480
|
+
return mapEdgePushItem(row);
|
|
481
|
+
}
|
|
482
|
+
nodeById(id) {
|
|
483
|
+
return this.db.prepare("SELECT * FROM node WHERE id = ?").get(id);
|
|
484
|
+
}
|
|
485
|
+
nextTimestamp() {
|
|
486
|
+
const ms = Math.max(Date.now(), this.lastTimestampMs + 1);
|
|
487
|
+
this.lastTimestampMs = ms;
|
|
488
|
+
return new Date(ms).toISOString();
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
export function openDaemonGraphStore(databaseFile) {
|
|
492
|
+
mkdirSync(dirname(databaseFile), { recursive: true });
|
|
493
|
+
return new DaemonGraphStore(new Database(databaseFile));
|
|
494
|
+
}
|