@nothumanwork/nn 0.1.1 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +8 -8
- package/src/db/client.ts +79 -41
- package/src/db/migrate.ts +29 -9
- package/src/db/replica-identity.ts +62 -0
- package/src/export/registry.ts +48 -46
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@nothumanwork/nn",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.3",
|
|
4
4
|
"description": "Neural Net — multi-provider transcript fabric for AI coding agents",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -53,13 +53,13 @@
|
|
|
53
53
|
"@libsql/client": "^0.17.3"
|
|
54
54
|
},
|
|
55
55
|
"optionalDependencies": {
|
|
56
|
-
"@nothumanwork/nn-darwin-arm64": "0.1.
|
|
57
|
-
"@nothumanwork/nn-darwin-x64": "0.1.
|
|
58
|
-
"@nothumanwork/nn-linux-arm64-gnu": "0.1.
|
|
59
|
-
"@nothumanwork/nn-linux-arm64-musl": "0.1.
|
|
60
|
-
"@nothumanwork/nn-linux-x64-gnu": "0.1.
|
|
61
|
-
"@nothumanwork/nn-linux-x64-musl": "0.1.
|
|
62
|
-
"@nothumanwork/nn-win32-x64": "0.1.
|
|
56
|
+
"@nothumanwork/nn-darwin-arm64": "0.1.3",
|
|
57
|
+
"@nothumanwork/nn-darwin-x64": "0.1.3",
|
|
58
|
+
"@nothumanwork/nn-linux-arm64-gnu": "0.1.3",
|
|
59
|
+
"@nothumanwork/nn-linux-arm64-musl": "0.1.3",
|
|
60
|
+
"@nothumanwork/nn-linux-x64-gnu": "0.1.3",
|
|
61
|
+
"@nothumanwork/nn-linux-x64-musl": "0.1.3",
|
|
62
|
+
"@nothumanwork/nn-win32-x64": "0.1.3"
|
|
63
63
|
},
|
|
64
64
|
"devDependencies": {
|
|
65
65
|
"@libsql/darwin-arm64": "0.5.29",
|
package/src/db/client.ts
CHANGED
|
@@ -235,33 +235,53 @@ export async function getSessionEvents(sessionId: string): Promise<
|
|
|
235
235
|
sourceSessionId: string;
|
|
236
236
|
}>
|
|
237
237
|
> {
|
|
238
|
-
return withDb(async (client) =>
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
238
|
+
return withDb(async (client) => getSessionEventsWithClient(client, sessionId));
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
export async function getSessionEventsWithClient(
|
|
242
|
+
client: Client,
|
|
243
|
+
sessionId: string,
|
|
244
|
+
): Promise<
|
|
245
|
+
Array<{
|
|
246
|
+
seq: number;
|
|
247
|
+
boundary: string;
|
|
248
|
+
role: string;
|
|
249
|
+
timestamp: string;
|
|
250
|
+
contentText: string;
|
|
251
|
+
toolName: string | null;
|
|
252
|
+
toolCallId: string | null;
|
|
253
|
+
toolInputJson: string | null;
|
|
254
|
+
toolOutputJson: string | null;
|
|
255
|
+
rawJson: string;
|
|
256
|
+
source: ProviderId;
|
|
257
|
+
sourceSessionId: string;
|
|
258
|
+
}>
|
|
259
|
+
> {
|
|
260
|
+
const result = await client.execute({
|
|
261
|
+
sql: `SELECT e.seq, e.boundary, e.role, e.timestamp, e.content_text, e.tool_name,
|
|
262
|
+
e.tool_call_id, e.tool_input_json, e.tool_output_json, e.raw_json,
|
|
263
|
+
s.source, s.source_session_id
|
|
264
|
+
FROM events e
|
|
265
|
+
JOIN sessions s ON s.id = e.session_id
|
|
266
|
+
WHERE e.session_id = ?
|
|
267
|
+
ORDER BY e.seq ASC`,
|
|
268
|
+
args: [sessionId],
|
|
264
269
|
});
|
|
270
|
+
return result.rows.map((row) => ({
|
|
271
|
+
seq: Number(row.seq),
|
|
272
|
+
boundary: String(row.boundary),
|
|
273
|
+
role: String(row.role),
|
|
274
|
+
timestamp: String(row.timestamp),
|
|
275
|
+
contentText: String(row.content_text),
|
|
276
|
+
toolName: row.tool_name ? String(row.tool_name) : null,
|
|
277
|
+
toolCallId: row.tool_call_id ? String(row.tool_call_id) : null,
|
|
278
|
+
toolInputJson: row.tool_input_json ? String(row.tool_input_json) : null,
|
|
279
|
+
toolOutputJson:
|
|
280
|
+
row.tool_output_json ? String(row.tool_output_json) : null,
|
|
281
|
+
rawJson: String(row.raw_json),
|
|
282
|
+
source: String(row.source) as ProviderId,
|
|
283
|
+
sourceSessionId: String(row.source_session_id),
|
|
284
|
+
}));
|
|
265
285
|
}
|
|
266
286
|
|
|
267
287
|
export async function searchEvents(
|
|
@@ -301,21 +321,39 @@ export async function recordExport(
|
|
|
301
321
|
lossy: boolean,
|
|
302
322
|
warnings: string[],
|
|
303
323
|
): Promise<void> {
|
|
304
|
-
await withDb(async (client) =>
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
324
|
+
await withDb(async (client) =>
|
|
325
|
+
recordExportWithClient(
|
|
326
|
+
client,
|
|
327
|
+
sessionId,
|
|
328
|
+
targetProvider,
|
|
329
|
+
outputPath,
|
|
330
|
+
lossy,
|
|
331
|
+
warnings,
|
|
332
|
+
),
|
|
333
|
+
);
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
export async function recordExportWithClient(
|
|
337
|
+
client: Client,
|
|
338
|
+
sessionId: string,
|
|
339
|
+
targetProvider: string,
|
|
340
|
+
outputPath: string,
|
|
341
|
+
lossy: boolean,
|
|
342
|
+
warnings: string[],
|
|
343
|
+
): Promise<void> {
|
|
344
|
+
const id = `exp:v1:${contentHash(`${sessionId}|${targetProvider}|${outputPath}|${Date.now()}`)}`;
|
|
345
|
+
await client.execute({
|
|
346
|
+
sql: `INSERT INTO exports(id, session_id, target_provider, output_path, exported_at, lossy, warnings_json)
|
|
347
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
|
348
|
+
args: [
|
|
349
|
+
id,
|
|
350
|
+
sessionId,
|
|
351
|
+
targetProvider,
|
|
352
|
+
outputPath,
|
|
353
|
+
Date.now(),
|
|
354
|
+
lossy ? 1 : 0,
|
|
355
|
+
JSON.stringify(warnings),
|
|
356
|
+
],
|
|
319
357
|
});
|
|
320
358
|
}
|
|
321
359
|
|
package/src/db/migrate.ts
CHANGED
|
@@ -5,6 +5,7 @@ import { dirname } from "node:path";
|
|
|
5
5
|
import { syncConfig, tursoConfigured } from "../config/sync.ts";
|
|
6
6
|
import { dbPath } from "../config/paths.ts";
|
|
7
7
|
import { checksum, readSchemaSql, withIngestLock } from "./lock.ts";
|
|
8
|
+
import { ensureLocalReplicaIdentity } from "./replica-identity.ts";
|
|
8
9
|
import {
|
|
9
10
|
findPendingBackup,
|
|
10
11
|
importedBackupDir,
|
|
@@ -44,15 +45,7 @@ export async function withDb<T>(
|
|
|
44
45
|
): Promise<T> {
|
|
45
46
|
return withIngestLock(async () => {
|
|
46
47
|
if (tursoConfigured()) {
|
|
47
|
-
|
|
48
|
-
activeDbClient = client;
|
|
49
|
-
try {
|
|
50
|
-
return await fn(client);
|
|
51
|
-
} finally {
|
|
52
|
-
client.close();
|
|
53
|
-
activeDbClient = null;
|
|
54
|
-
resetDbCache();
|
|
55
|
-
}
|
|
48
|
+
return runEmbeddedReplicaOp(fn);
|
|
56
49
|
}
|
|
57
50
|
|
|
58
51
|
const client = await getDb();
|
|
@@ -60,6 +53,31 @@ export async function withDb<T>(
|
|
|
60
53
|
});
|
|
61
54
|
}
|
|
62
55
|
|
|
56
|
+
async function runEmbeddedReplicaOp<T>(
|
|
57
|
+
fn: (client: Client) => Promise<T>,
|
|
58
|
+
allowRecovery = true,
|
|
59
|
+
): Promise<T> {
|
|
60
|
+
const path = dbPath();
|
|
61
|
+
|
|
62
|
+
try {
|
|
63
|
+
const client = await openDbClient(path, false);
|
|
64
|
+
activeDbClient = client;
|
|
65
|
+
try {
|
|
66
|
+
return await fn(client);
|
|
67
|
+
} finally {
|
|
68
|
+
client.close();
|
|
69
|
+
activeDbClient = null;
|
|
70
|
+
resetDbCache();
|
|
71
|
+
}
|
|
72
|
+
} catch (error) {
|
|
73
|
+
if (allowRecovery && isReplicaConflictError(error)) {
|
|
74
|
+
recoverEmbeddedReplica(path);
|
|
75
|
+
return runEmbeddedReplicaOp(fn, false);
|
|
76
|
+
}
|
|
77
|
+
throw error;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
63
81
|
export async function getDb(): Promise<Client> {
|
|
64
82
|
if (activeDbClient) {
|
|
65
83
|
return activeDbClient;
|
|
@@ -76,6 +94,8 @@ async function openDbClient(
|
|
|
76
94
|
path = dbPath(),
|
|
77
95
|
allowRecovery = true,
|
|
78
96
|
): Promise<Client> {
|
|
97
|
+
ensureLocalReplicaIdentity(path);
|
|
98
|
+
|
|
79
99
|
try {
|
|
80
100
|
const client = createDbClient(path);
|
|
81
101
|
try {
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { hostname } from "node:os";
|
|
3
|
+
import { dirname } from "node:path";
|
|
4
|
+
|
|
5
|
+
import { recoverEmbeddedReplica, replicaInfoPath } from "./replica-migrate.ts";
|
|
6
|
+
|
|
7
|
+
const IDENTITY_BASENAME = "replica-identity.json";
|
|
8
|
+
|
|
9
|
+
interface ReplicaIdentity {
|
|
10
|
+
host: string;
|
|
11
|
+
pid: number;
|
|
12
|
+
updatedAt: number;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function identityPath(dbPath: string): string {
|
|
16
|
+
return `${dirname(dbPath)}/${IDENTITY_BASENAME}`;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function readIdentity(dbPath: string): ReplicaIdentity | null {
|
|
20
|
+
const path = identityPath(dbPath);
|
|
21
|
+
if (!existsSync(path)) {
|
|
22
|
+
return null;
|
|
23
|
+
}
|
|
24
|
+
try {
|
|
25
|
+
return JSON.parse(readFileSync(path, "utf-8")) as ReplicaIdentity;
|
|
26
|
+
} catch {
|
|
27
|
+
return null;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function writeIdentity(dbPath: string): void {
|
|
32
|
+
const path = identityPath(dbPath);
|
|
33
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
34
|
+
const identity: ReplicaIdentity = {
|
|
35
|
+
host: hostname(),
|
|
36
|
+
pid: process.pid,
|
|
37
|
+
updatedAt: Date.now(),
|
|
38
|
+
};
|
|
39
|
+
writeFileSync(path, `${JSON.stringify(identity, null, 2)}\n`, "utf-8");
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** Reset embedded replica when local files came from another machine (e.g. git clone). */
|
|
43
|
+
export function ensureLocalReplicaIdentity(dbPath: string): boolean {
|
|
44
|
+
if (!existsSync(replicaInfoPath(dbPath))) {
|
|
45
|
+
writeIdentity(dbPath);
|
|
46
|
+
return false;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
const identity = readIdentity(dbPath);
|
|
50
|
+
const host = hostname();
|
|
51
|
+
if (identity && identity.host !== host) {
|
|
52
|
+
console.error(
|
|
53
|
+
`[nn-db] embedded replica belongs to host "${identity.host}"; resetting for "${host}"`,
|
|
54
|
+
);
|
|
55
|
+
recoverEmbeddedReplica(dbPath);
|
|
56
|
+
writeIdentity(dbPath);
|
|
57
|
+
return true;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
writeIdentity(dbPath);
|
|
61
|
+
return false;
|
|
62
|
+
}
|
package/src/export/registry.ts
CHANGED
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
import { mkdirSync, writeFileSync } from "node:fs";
|
|
2
2
|
import { dirname } from "node:path";
|
|
3
3
|
|
|
4
|
-
import {
|
|
5
|
-
|
|
4
|
+
import {
|
|
5
|
+
getSessionEventsWithClient,
|
|
6
|
+
recordExportWithClient,
|
|
7
|
+
} from "../db/client.ts";
|
|
6
8
|
import { findSessionBySourceId } from "../db/client.ts";
|
|
9
|
+
import { withDb } from "../db/migrate.ts";
|
|
7
10
|
import type { ProviderId } from "../ir/types.ts";
|
|
8
11
|
import { claudeExporter } from "./claude.ts";
|
|
9
12
|
import { codexExporter } from "./codex.ts";
|
|
@@ -47,61 +50,60 @@ export async function exportSession(
|
|
|
47
50
|
manifestPath: string | null;
|
|
48
51
|
manifest: ExportManifest;
|
|
49
52
|
}> {
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
53
|
+
return withDb(async (client) => {
|
|
54
|
+
const events = await getSessionEventsWithClient(client, sessionId);
|
|
55
|
+
if (events.length === 0) {
|
|
56
|
+
throw new Error(`No events found for session ${sessionId}`);
|
|
57
|
+
}
|
|
54
58
|
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
59
|
+
const resolvedTarget = target ?? events[0]!.source;
|
|
60
|
+
const exporter = getExporter(resolvedTarget);
|
|
61
|
+
const result = exporter.export({
|
|
62
|
+
sessionId,
|
|
63
|
+
source: events[0]!.source,
|
|
64
|
+
sourceSessionId: events[0]!.sourceSessionId,
|
|
65
|
+
events,
|
|
66
|
+
});
|
|
63
67
|
|
|
64
|
-
|
|
68
|
+
const content =
|
|
69
|
+
result.lines.length > 0 ? `${result.lines.join("\n")}\n` : "";
|
|
65
70
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
71
|
+
const manifest: ExportManifest = {
|
|
72
|
+
source: events[0]!.source,
|
|
73
|
+
target: resolvedTarget,
|
|
74
|
+
sessionId,
|
|
75
|
+
sourceSessionId: events[0]!.sourceSessionId,
|
|
76
|
+
lossy: result.lossy,
|
|
77
|
+
warnings: result.warnings.map((warning) => warning.message),
|
|
78
|
+
exportedAt: new Date().toISOString(),
|
|
79
|
+
lineCount: result.lines.length,
|
|
80
|
+
};
|
|
76
81
|
|
|
77
|
-
|
|
78
|
-
await
|
|
82
|
+
const resolvedOutputPath = outputPath ?? "-";
|
|
83
|
+
await recordExportWithClient(
|
|
84
|
+
client,
|
|
79
85
|
sessionId,
|
|
80
86
|
resolvedTarget,
|
|
81
|
-
|
|
87
|
+
resolvedOutputPath,
|
|
82
88
|
result.lossy,
|
|
83
89
|
manifest.warnings,
|
|
84
90
|
);
|
|
85
|
-
return { content, outputPath: null, manifestPath: null, manifest };
|
|
86
|
-
}
|
|
87
91
|
|
|
88
|
-
|
|
89
|
-
|
|
92
|
+
if (!outputPath) {
|
|
93
|
+
return { content, outputPath: null, manifestPath: null, manifest };
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
mkdirSync(dirname(outputPath), { recursive: true });
|
|
97
|
+
writeFileSync(outputPath, content, "utf-8");
|
|
90
98
|
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
resolvedTarget,
|
|
100
|
-
outputPath,
|
|
101
|
-
result.lossy,
|
|
102
|
-
manifest.warnings,
|
|
103
|
-
);
|
|
104
|
-
return { content, outputPath, manifestPath, manifest };
|
|
99
|
+
const manifestPath = manifestPathFor(outputPath);
|
|
100
|
+
writeFileSync(
|
|
101
|
+
manifestPath,
|
|
102
|
+
`${JSON.stringify(manifest, null, 2)}\n`,
|
|
103
|
+
"utf-8",
|
|
104
|
+
);
|
|
105
|
+
return { content, outputPath, manifestPath, manifest };
|
|
106
|
+
});
|
|
105
107
|
}
|
|
106
108
|
|
|
107
109
|
export async function resolveSessionId(
|