@inixiative/archive 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +100 -0
- package/package.json +57 -0
- package/src/cli.ts +238 -0
- package/src/client.ts +148 -0
- package/src/collector.ts +85 -0
- package/src/config.ts +80 -0
- package/src/import-file.ts +45 -0
- package/src/import.ts +173 -0
- package/src/index.ts +172 -0
- package/src/local.ts +150 -0
- package/src/preview.ts +57 -0
- package/src/server.ts +155 -0
package/src/config.ts
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { chmodSync, existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
|
|
3
|
+
import { dirname } from 'node:path';
|
|
4
|
+
import { z } from 'zod';
|
|
5
|
+
|
|
6
|
+
const common = {
|
|
7
|
+
projectId: z.string().min(1).max(256),
|
|
8
|
+
url: z.url(),
|
|
9
|
+
tokenEnv: z.string().regex(/^[A-Z][A-Z0-9_]+$/),
|
|
10
|
+
};
|
|
11
|
+
export const archiveDestinationSchema = z.union([
|
|
12
|
+
z.strictObject({
|
|
13
|
+
...common,
|
|
14
|
+
kind: z.literal('archive'),
|
|
15
|
+
keepIds: z.array(z.string()).max(0).default([]),
|
|
16
|
+
}),
|
|
17
|
+
z.strictObject({
|
|
18
|
+
...common,
|
|
19
|
+
kind: z.literal('kingdom').optional(),
|
|
20
|
+
kastleId: z.uuid(),
|
|
21
|
+
keepIds: z.array(z.uuid()).max(50).default([]),
|
|
22
|
+
}),
|
|
23
|
+
]);
|
|
24
|
+
export type ArchiveDestination = z.infer<typeof archiveDestinationSchema>;
|
|
25
|
+
|
|
26
|
+
export function destinationUrl(input: string) {
|
|
27
|
+
const url = new URL(input);
|
|
28
|
+
if (
|
|
29
|
+
url.username ||
|
|
30
|
+
url.password ||
|
|
31
|
+
url.search ||
|
|
32
|
+
url.hash ||
|
|
33
|
+
(url.protocol !== 'https:' &&
|
|
34
|
+
!(url.protocol === 'http:' && ['localhost', '127.0.0.1', '[::1]'].includes(url.hostname)))
|
|
35
|
+
)
|
|
36
|
+
throw new Error('Archive destination requires HTTPS or loopback HTTP and no URL credentials');
|
|
37
|
+
if (!url.pathname.endsWith('/')) url.pathname += '/';
|
|
38
|
+
return url;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export function readDestinations(file: string): ArchiveDestination[] {
|
|
42
|
+
return existsSync(file)
|
|
43
|
+
? z
|
|
44
|
+
.array(archiveDestinationSchema)
|
|
45
|
+
.max(100)
|
|
46
|
+
.parse(JSON.parse(readFileSync(file, 'utf8')))
|
|
47
|
+
: [];
|
|
48
|
+
}
|
|
49
|
+
export function destinationIdentity(destination: ArchiveDestination) {
|
|
50
|
+
const url = destinationUrl(destination.url);
|
|
51
|
+
// Preserve existing Foundry/Kingdom upload receipt identities.
|
|
52
|
+
return JSON.stringify([
|
|
53
|
+
url.origin,
|
|
54
|
+
new URL(destination.url).pathname,
|
|
55
|
+
destination.kind === 'archive' ? 'standalone' : destination.kastleId,
|
|
56
|
+
]);
|
|
57
|
+
}
|
|
58
|
+
export function connectDestination(file: string, input: unknown) {
|
|
59
|
+
const destination = archiveDestinationSchema.parse(input);
|
|
60
|
+
destination.url = destinationUrl(destination.url).href;
|
|
61
|
+
const destinations = readDestinations(file);
|
|
62
|
+
const identity = destinationIdentity(destination);
|
|
63
|
+
const index = destinations.findIndex(
|
|
64
|
+
(item) => item.projectId === destination.projectId && destinationIdentity(item) === identity,
|
|
65
|
+
);
|
|
66
|
+
if (index < 0) destinations.push(destination);
|
|
67
|
+
else destinations[index] = destination;
|
|
68
|
+
mkdirSync(dirname(file), { recursive: true, mode: 0o700 });
|
|
69
|
+
const temp = `${file}.${randomUUID()}.tmp`;
|
|
70
|
+
writeFileSync(temp, JSON.stringify(destinations, null, 2) + '\n', { mode: 0o600, flag: 'wx' });
|
|
71
|
+
renameSync(temp, file);
|
|
72
|
+
chmodSync(file, 0o600);
|
|
73
|
+
return {
|
|
74
|
+
configured: true,
|
|
75
|
+
kind: destination.kind ?? 'kingdom',
|
|
76
|
+
projectId: destination.projectId,
|
|
77
|
+
url: destination.url,
|
|
78
|
+
tokenEnv: destination.tokenEnv,
|
|
79
|
+
};
|
|
80
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { closeSync, fstatSync, openSync, readSync } from 'node:fs';
|
|
2
|
+
import { type ImportOptions, importTranscriptLines } from './import';
|
|
3
|
+
|
|
4
|
+
/** Bound raw record memory while skipping unsupported payloads from long native histories. */
|
|
5
|
+
export function importTranscriptFile(file: string, options: ImportOptions) {
|
|
6
|
+
function* lines() {
|
|
7
|
+
const descriptor = openSync(file, 'r');
|
|
8
|
+
try {
|
|
9
|
+
const stat = fstatSync(descriptor);
|
|
10
|
+
if (!stat.isFile() || stat.size > 1_000_000_000)
|
|
11
|
+
throw new Error('Import requires a transcript file no larger than 1 GB');
|
|
12
|
+
const buffer = Buffer.alloc(64 * 1024);
|
|
13
|
+
const decoder = new TextDecoder('utf-8', { fatal: true });
|
|
14
|
+
let pending = '',
|
|
15
|
+
bytes = 0;
|
|
16
|
+
while (true) {
|
|
17
|
+
const count = readSync(descriptor, buffer, 0, buffer.length, null);
|
|
18
|
+
bytes += count;
|
|
19
|
+
if (bytes > 1_000_000_000) throw new Error('Transcript grew beyond the 1 GB import limit');
|
|
20
|
+
pending += count
|
|
21
|
+
? decoder.decode(buffer.subarray(0, count), { stream: true })
|
|
22
|
+
: decoder.decode();
|
|
23
|
+
let start = 0;
|
|
24
|
+
let newline = pending.indexOf('\n', start);
|
|
25
|
+
while (newline >= 0) {
|
|
26
|
+
if (newline - start > 64_000_000)
|
|
27
|
+
throw new Error('Transcript record exceeds the 64 million character limit');
|
|
28
|
+
yield pending.slice(start, newline);
|
|
29
|
+
start = newline + 1;
|
|
30
|
+
newline = pending.indexOf('\n', start);
|
|
31
|
+
}
|
|
32
|
+
pending = pending.slice(start);
|
|
33
|
+
if (pending.length > 64_000_000)
|
|
34
|
+
throw new Error('Transcript record exceeds the 64 million character limit');
|
|
35
|
+
if (!count) {
|
|
36
|
+
if (pending) yield pending;
|
|
37
|
+
break;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
} finally {
|
|
41
|
+
closeSync(descriptor);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
return importTranscriptLines(lines(), options);
|
|
45
|
+
}
|
package/src/import.ts
ADDED
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
import { type ArchiveEntry, type ArchiveSnapshot, archiveSnapshotSchema } from './index';
|
|
2
|
+
|
|
3
|
+
export interface ImportOptions {
|
|
4
|
+
source: 'codex' | 'claude-code';
|
|
5
|
+
sourceId: string;
|
|
6
|
+
sessionId?: string;
|
|
7
|
+
title?: string;
|
|
8
|
+
projectId?: string;
|
|
9
|
+
}
|
|
10
|
+
export function importTranscript(text: string, options: ImportOptions): ArchiveSnapshot {
|
|
11
|
+
return importTranscriptLines(text.split('\n'), options);
|
|
12
|
+
}
|
|
13
|
+
export function importTranscriptLines(
|
|
14
|
+
lines: Iterable<string>,
|
|
15
|
+
options: ImportOptions,
|
|
16
|
+
): ArchiveSnapshot {
|
|
17
|
+
const entries: ArchiveEntry[] = [];
|
|
18
|
+
const omissions = new Set<string>();
|
|
19
|
+
let sessionId = options.sessionId;
|
|
20
|
+
let reasoning = false;
|
|
21
|
+
const seen = new Map<string, string>();
|
|
22
|
+
let index = -1;
|
|
23
|
+
let contentBytes = 0;
|
|
24
|
+
for (const rawLine of lines) {
|
|
25
|
+
index++;
|
|
26
|
+
const line = rawLine.trim();
|
|
27
|
+
if (!line) continue;
|
|
28
|
+
let row: any;
|
|
29
|
+
try {
|
|
30
|
+
row = JSON.parse(line);
|
|
31
|
+
} catch {
|
|
32
|
+
throw new Error(`Invalid transcript JSON at line ${index + 1}; no archive imported`);
|
|
33
|
+
}
|
|
34
|
+
if (!row || typeof row !== 'object' || Array.isArray(row))
|
|
35
|
+
throw new Error(`Invalid transcript record at line ${index + 1}`);
|
|
36
|
+
if (options.source === 'codex' && row.type === 'session_meta') {
|
|
37
|
+
const id = row.payload?.id;
|
|
38
|
+
if (sessionId && id && id !== sessionId)
|
|
39
|
+
throw new Error('Transcript session identity mismatch');
|
|
40
|
+
sessionId ??= id;
|
|
41
|
+
}
|
|
42
|
+
if (options.source === 'claude-code' && row.sessionId) {
|
|
43
|
+
if (sessionId && sessionId !== row.sessionId) throw new Error('Mixed transcript sessions');
|
|
44
|
+
sessionId = row.sessionId;
|
|
45
|
+
}
|
|
46
|
+
const timestamp = typeof row.timestamp === 'number' ? row.timestamp : Date.parse(row.timestamp);
|
|
47
|
+
const add = (kind: ArchiveEntry['kind'], content: unknown, suffix: string, callId?: string) => {
|
|
48
|
+
if (typeof content !== 'string' || !content.length) return;
|
|
49
|
+
const id = `${row.uuid ?? row.payload?.id ?? `line-${index + 1}`}:${suffix}`;
|
|
50
|
+
const identity = JSON.stringify([kind, content, callId]);
|
|
51
|
+
if (seen.has(id)) {
|
|
52
|
+
if (seen.get(id) !== identity)
|
|
53
|
+
throw new Error(`Conflicting transcript record at line ${index + 1}`);
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
contentBytes += Buffer.byteLength(content);
|
|
57
|
+
if (contentBytes > 64_000_000 || entries.length >= 50_000 || content.length > 2_000_000)
|
|
58
|
+
throw new Error(
|
|
59
|
+
'Supported transcript content exceeds the archive size limit; no archive imported',
|
|
60
|
+
);
|
|
61
|
+
seen.set(id, identity);
|
|
62
|
+
entries.push({
|
|
63
|
+
id,
|
|
64
|
+
kind,
|
|
65
|
+
text: content,
|
|
66
|
+
timestamp: Number.isFinite(timestamp) ? timestamp : null,
|
|
67
|
+
sourceRef: `line:${index + 1}`,
|
|
68
|
+
...(callId ? { callId } : {}),
|
|
69
|
+
});
|
|
70
|
+
};
|
|
71
|
+
if (options.source === 'codex') {
|
|
72
|
+
if (row.type === 'response_item') {
|
|
73
|
+
const p = row.payload ?? {};
|
|
74
|
+
if (p.type === 'message' && ['user', 'assistant'].includes(p.role)) {
|
|
75
|
+
for (const [i, block] of (p.content ?? []).entries()) {
|
|
76
|
+
if (['input_text', 'output_text', 'text'].includes(block.type))
|
|
77
|
+
add(p.role, block.text, `text-${i}`);
|
|
78
|
+
else omissions.add('Non-text message content is not included');
|
|
79
|
+
}
|
|
80
|
+
} else if (['function_call', 'custom_tool_call'].includes(p.type)) {
|
|
81
|
+
add(
|
|
82
|
+
'tool-call',
|
|
83
|
+
JSON.stringify({ name: p.name, arguments: p.arguments ?? p.input }),
|
|
84
|
+
'call',
|
|
85
|
+
p.call_id,
|
|
86
|
+
);
|
|
87
|
+
} else if (['function_call_output', 'custom_tool_call_output'].includes(p.type)) {
|
|
88
|
+
const output =
|
|
89
|
+
typeof p.output === 'string'
|
|
90
|
+
? p.output
|
|
91
|
+
: Array.isArray(p.output)
|
|
92
|
+
? p.output
|
|
93
|
+
.filter((block: any) =>
|
|
94
|
+
['text', 'input_text', 'output_text'].includes(block.type),
|
|
95
|
+
)
|
|
96
|
+
.map((block: any) => block.text)
|
|
97
|
+
.join('\n')
|
|
98
|
+
: '';
|
|
99
|
+
if (typeof p.output !== 'string')
|
|
100
|
+
omissions.add('Only text tool output blocks are included');
|
|
101
|
+
add('tool-result', output, 'result', p.call_id);
|
|
102
|
+
} else if (p.type === 'reasoning') {
|
|
103
|
+
for (const [i, block] of (p.summary ?? []).entries()) {
|
|
104
|
+
if (typeof block.text === 'string') {
|
|
105
|
+
add('reasoning-summary', block.text, `summary-${i}`);
|
|
106
|
+
reasoning = true;
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
omissions.add('Private reasoning and encrypted payloads are not included');
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
} else if (['user', 'assistant'].includes(row.type) && row.message) {
|
|
113
|
+
const blocks = row.message.content;
|
|
114
|
+
if (typeof blocks === 'string') add(row.type, blocks, 'text');
|
|
115
|
+
else if (Array.isArray(blocks))
|
|
116
|
+
for (const [i, block] of blocks.entries()) {
|
|
117
|
+
if (block.type === 'text') add(row.type, block.text, `text-${i}`);
|
|
118
|
+
else if (block.type === 'tool_use')
|
|
119
|
+
add(
|
|
120
|
+
'tool-call',
|
|
121
|
+
JSON.stringify({ name: block.name, input: block.input }),
|
|
122
|
+
`call-${i}`,
|
|
123
|
+
block.id,
|
|
124
|
+
);
|
|
125
|
+
else if (block.type === 'tool_result') {
|
|
126
|
+
const output =
|
|
127
|
+
typeof block.content === 'string'
|
|
128
|
+
? block.content
|
|
129
|
+
: Array.isArray(block.content)
|
|
130
|
+
? block.content
|
|
131
|
+
.filter((b: any) => b.type === 'text')
|
|
132
|
+
.map((b: any) => b.text)
|
|
133
|
+
.join('\n')
|
|
134
|
+
: '';
|
|
135
|
+
add('tool-result', output, `result-${i}`, block.tool_use_id);
|
|
136
|
+
if (Array.isArray(block.content) && block.content.some((b: any) => b.type !== 'text'))
|
|
137
|
+
omissions.add('Non-text tool content is not included');
|
|
138
|
+
} else if (['thinking', 'redacted_thinking'].includes(block.type))
|
|
139
|
+
omissions.add('Private reasoning is not included');
|
|
140
|
+
else omissions.add('Non-text message content is not included');
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
if (!sessionId || !entries.length)
|
|
145
|
+
throw new Error('Transcript must identify a session and contain supported records');
|
|
146
|
+
const firstPrompt = entries.find(
|
|
147
|
+
(entry) =>
|
|
148
|
+
entry.kind === 'user' &&
|
|
149
|
+
!/^<(?:recommended_plugins|environment_context|user_instructions|permissions)(?:>|\s)/.test(
|
|
150
|
+
entry.text.trimStart(),
|
|
151
|
+
),
|
|
152
|
+
);
|
|
153
|
+
const title = options.title ?? firstPrompt?.text.slice(0, 160) ?? sessionId;
|
|
154
|
+
return archiveSnapshotSchema.parse({
|
|
155
|
+
schemaVersion: 1,
|
|
156
|
+
sourceId: options.sourceId,
|
|
157
|
+
source: options.source,
|
|
158
|
+
sessionId,
|
|
159
|
+
title,
|
|
160
|
+
projectId: options.projectId,
|
|
161
|
+
tags: [],
|
|
162
|
+
capturedAt: Date.now(),
|
|
163
|
+
coverage: {
|
|
164
|
+
reasoning: reasoning ? 'summaries-only' : 'unavailable',
|
|
165
|
+
completeness: 'partial',
|
|
166
|
+
omissions: [
|
|
167
|
+
'Import contains public transcript records only; source history may be incomplete',
|
|
168
|
+
...omissions,
|
|
169
|
+
],
|
|
170
|
+
},
|
|
171
|
+
entries,
|
|
172
|
+
});
|
|
173
|
+
}
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { getEncoding } from 'js-tiktoken';
|
|
3
|
+
import { z } from 'zod';
|
|
4
|
+
|
|
5
|
+
const identifier = z.string().min(1).max(256);
|
|
6
|
+
export const archiveEntrySchema = z.strictObject({
|
|
7
|
+
id: identifier,
|
|
8
|
+
turnId: identifier.optional(),
|
|
9
|
+
callId: identifier.optional(),
|
|
10
|
+
kind: z.enum(['user', 'assistant', 'tool-call', 'tool-result', 'reasoning-summary', 'event']),
|
|
11
|
+
text: z.string().max(2_000_000),
|
|
12
|
+
timestamp: z.number().finite().nonnegative().nullable(),
|
|
13
|
+
sourceRef: z.string().min(1).max(512),
|
|
14
|
+
});
|
|
15
|
+
export const archiveSnapshotSchema = z
|
|
16
|
+
.strictObject({
|
|
17
|
+
schemaVersion: z.literal(1),
|
|
18
|
+
sourceId: z.uuid(),
|
|
19
|
+
source: z.enum(['foundry', 'claude-code', 'codex']),
|
|
20
|
+
sessionId: identifier,
|
|
21
|
+
title: z.string().min(1).max(500),
|
|
22
|
+
projectId: identifier.optional(),
|
|
23
|
+
tags: z.array(z.string().min(1).max(120)).max(100),
|
|
24
|
+
goalIds: z.array(identifier).max(100).default([]),
|
|
25
|
+
runIds: z.array(identifier).max(100).default([]),
|
|
26
|
+
capturedAt: z.number().finite().nonnegative(),
|
|
27
|
+
coverage: z.strictObject({
|
|
28
|
+
reasoning: z.enum(['unavailable', 'summaries-only']),
|
|
29
|
+
completeness: z.enum(['recorded', 'partial']),
|
|
30
|
+
omissions: z.array(z.string().max(500)).max(100),
|
|
31
|
+
}),
|
|
32
|
+
entries: z.array(archiveEntrySchema).max(50_000),
|
|
33
|
+
})
|
|
34
|
+
.superRefine((value, ctx) => {
|
|
35
|
+
if (new Set(value.entries.map((entry) => entry.id)).size !== value.entries.length)
|
|
36
|
+
ctx.addIssue({ code: 'custom', message: 'Duplicate archive entry identities' });
|
|
37
|
+
if (Buffer.byteLength(JSON.stringify(value)) > 64_000_000)
|
|
38
|
+
ctx.addIssue({ code: 'custom', message: 'Archive exceeds 64 MB' });
|
|
39
|
+
});
|
|
40
|
+
export type ArchiveSnapshot = z.infer<typeof archiveSnapshotSchema>;
|
|
41
|
+
export type ArchiveEntry = z.infer<typeof archiveEntrySchema>;
|
|
42
|
+
export interface ArchiveChunk {
|
|
43
|
+
id: string;
|
|
44
|
+
entryId: string;
|
|
45
|
+
sourceRef: string;
|
|
46
|
+
kind: ArchiveEntry['kind'];
|
|
47
|
+
start: number;
|
|
48
|
+
end: number;
|
|
49
|
+
text: string;
|
|
50
|
+
tokenCount: number;
|
|
51
|
+
encoding: 'cl100k_base';
|
|
52
|
+
}
|
|
53
|
+
let encoding: ReturnType<typeof getEncoding> | undefined;
|
|
54
|
+
export const tokenCount = (text: string) => {
|
|
55
|
+
encoding ??= getEncoding('cl100k_base');
|
|
56
|
+
return encoding.encode(text, [], []).length;
|
|
57
|
+
};
|
|
58
|
+
export const digest = (text: string) => createHash('sha256').update(text).digest('hex');
|
|
59
|
+
export const archiveKey = (snapshot: Pick<ArchiveSnapshot, 'sourceId' | 'source' | 'sessionId'>) =>
|
|
60
|
+
digest(JSON.stringify([snapshot.sourceId, snapshot.source, snapshot.sessionId]));
|
|
61
|
+
export const snapshotDigest = (snapshot: ArchiveSnapshot) => {
|
|
62
|
+
const { capturedAt: _, ...content } = snapshot;
|
|
63
|
+
return digest(JSON.stringify(content));
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
export function chunkArchive(snapshot: ArchiveSnapshot, budget = 512): ArchiveChunk[] {
|
|
67
|
+
if (!Number.isInteger(budget) || budget < 16 || budget > 8192)
|
|
68
|
+
throw new Error('Invalid chunk token budget');
|
|
69
|
+
const chunks: ArchiveChunk[] = [];
|
|
70
|
+
for (const entry of snapshot.entries) {
|
|
71
|
+
let start = 0;
|
|
72
|
+
while (start < entry.text.length) {
|
|
73
|
+
let end = Math.min(entry.text.length, start + budget * 3);
|
|
74
|
+
if (end < entry.text.length && /[\uD800-\uDBFF]/.test(entry.text[end - 1])) end--;
|
|
75
|
+
let text = entry.text.slice(start, end);
|
|
76
|
+
while (tokenCount(text) > budget) {
|
|
77
|
+
end = start + Math.max(1, Math.floor((end - start) * 0.8));
|
|
78
|
+
if (end < entry.text.length && /[\uD800-\uDBFF]/.test(entry.text[end - 1])) end--;
|
|
79
|
+
text = entry.text.slice(start, end);
|
|
80
|
+
}
|
|
81
|
+
chunks.push({
|
|
82
|
+
id: digest(JSON.stringify([entry.id, start, end, text])),
|
|
83
|
+
entryId: entry.id,
|
|
84
|
+
sourceRef: entry.sourceRef,
|
|
85
|
+
kind: entry.kind,
|
|
86
|
+
start,
|
|
87
|
+
end,
|
|
88
|
+
text,
|
|
89
|
+
tokenCount: tokenCount(text),
|
|
90
|
+
encoding: 'cl100k_base',
|
|
91
|
+
});
|
|
92
|
+
start = end;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
return chunks;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
export function selectChunks(chunks: ArchiveChunk[], query: string, budget = 2048) {
|
|
99
|
+
if (!Number.isInteger(budget) || budget < 16 || budget > 32768)
|
|
100
|
+
throw new Error('Invalid context token budget');
|
|
101
|
+
const terms = [...new Set(query.toLowerCase().match(/[\p{L}\p{N}_-]+/gu) ?? [])].slice(0, 64);
|
|
102
|
+
const ranked = chunks
|
|
103
|
+
.map((chunk, index) => ({
|
|
104
|
+
chunk,
|
|
105
|
+
index,
|
|
106
|
+
score: terms.reduce(
|
|
107
|
+
(score, term) => score + Number(chunk.text.toLowerCase().includes(term)),
|
|
108
|
+
0,
|
|
109
|
+
),
|
|
110
|
+
}))
|
|
111
|
+
.filter((item) => !terms.length || item.score > 0)
|
|
112
|
+
.sort((a, b) => b.score - a.score || a.index - b.index);
|
|
113
|
+
const selected: ArchiveChunk[] = [];
|
|
114
|
+
let used = 0;
|
|
115
|
+
for (const { chunk } of ranked) {
|
|
116
|
+
const remaining = budget - used;
|
|
117
|
+
if (remaining < 16) break;
|
|
118
|
+
if (chunk.tokenCount <= remaining) {
|
|
119
|
+
selected.push(chunk);
|
|
120
|
+
used += chunk.tokenCount;
|
|
121
|
+
continue;
|
|
122
|
+
}
|
|
123
|
+
// Return a source-addressable excerpt even when the requested context is smaller than a stored chunk.
|
|
124
|
+
const lower = chunk.text.toLowerCase();
|
|
125
|
+
const match =
|
|
126
|
+
terms
|
|
127
|
+
.map((term) => lower.indexOf(term))
|
|
128
|
+
.filter((index) => index >= 0)
|
|
129
|
+
.sort((a, b) => a - b)[0] ?? 0;
|
|
130
|
+
let offset = Math.max(0, match - Math.floor(remaining / 4));
|
|
131
|
+
if (offset > 0 && /[\uDC00-\uDFFF]/.test(chunk.text[offset])) offset--;
|
|
132
|
+
let end = Math.min(chunk.text.length, offset + remaining * 3);
|
|
133
|
+
if (end < chunk.text.length && /[\uD800-\uDBFF]/.test(chunk.text[end - 1])) end--;
|
|
134
|
+
while (tokenCount(chunk.text.slice(offset, end)) > remaining) {
|
|
135
|
+
end = offset + Math.max(1, Math.floor((end - offset) * 0.8));
|
|
136
|
+
if (end < chunk.text.length && /[\uD800-\uDBFF]/.test(chunk.text[end - 1])) end--;
|
|
137
|
+
}
|
|
138
|
+
const text = chunk.text.slice(offset, end);
|
|
139
|
+
selected.push({
|
|
140
|
+
...chunk,
|
|
141
|
+
id: digest(JSON.stringify([chunk.entryId, chunk.start + offset, chunk.start + end, text])),
|
|
142
|
+
start: chunk.start + offset,
|
|
143
|
+
end: chunk.start + end,
|
|
144
|
+
text,
|
|
145
|
+
tokenCount: tokenCount(text),
|
|
146
|
+
});
|
|
147
|
+
used += tokenCount(text);
|
|
148
|
+
}
|
|
149
|
+
return {
|
|
150
|
+
chunks: selected,
|
|
151
|
+
tokenCount: used,
|
|
152
|
+
encoding: 'cl100k_base' as const,
|
|
153
|
+
matchingChunks: ranked.length,
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
export function categorySuggestions(snapshot: ArchiveSnapshot) {
|
|
158
|
+
const text = snapshot.entries
|
|
159
|
+
.filter((entry) => entry.kind === 'user')
|
|
160
|
+
.map((entry) => entry.text)
|
|
161
|
+
.join('\n')
|
|
162
|
+
.toLowerCase();
|
|
163
|
+
const categories = {
|
|
164
|
+
debugging: /\b(bug|error|fix|failure|regression)\b/,
|
|
165
|
+
documentation: /\b(docs|documentation|readme)\b/,
|
|
166
|
+
testing: /\b(test|tests|verification|qa)\b/,
|
|
167
|
+
design: /\b(design|architecture|diagram|proposal)\b/,
|
|
168
|
+
};
|
|
169
|
+
return Object.entries(categories)
|
|
170
|
+
.filter(([, pattern]) => pattern.test(text))
|
|
171
|
+
.map(([tag]) => ({ tag, origin: 'heuristic' as const }));
|
|
172
|
+
}
|
package/src/local.ts
ADDED
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
import { Database } from 'bun:sqlite';
|
|
2
|
+
import { randomUUID } from 'node:crypto';
|
|
3
|
+
import { chmodSync, mkdirSync } from 'node:fs';
|
|
4
|
+
import { dirname } from 'node:path';
|
|
5
|
+
import {
|
|
6
|
+
type ArchiveChunk,
|
|
7
|
+
type ArchiveSnapshot,
|
|
8
|
+
archiveKey,
|
|
9
|
+
archiveSnapshotSchema,
|
|
10
|
+
categorySuggestions,
|
|
11
|
+
chunkArchive,
|
|
12
|
+
selectChunks,
|
|
13
|
+
snapshotDigest,
|
|
14
|
+
} from './index';
|
|
15
|
+
|
|
16
|
+
export class LocalArchiveStore {
|
|
17
|
+
private readonly db: Database;
|
|
18
|
+
readonly sourceId: string;
|
|
19
|
+
constructor(path: string) {
|
|
20
|
+
if (path !== ':memory:') mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
|
|
21
|
+
this.db = new Database(path, { create: true });
|
|
22
|
+
if (path !== ':memory:') chmodSync(path, 0o600);
|
|
23
|
+
this.db.exec(`PRAGMA foreign_keys=ON; PRAGMA busy_timeout=1000;
|
|
24
|
+
CREATE TABLE IF NOT EXISTS settings (key TEXT PRIMARY KEY, value TEXT NOT NULL);
|
|
25
|
+
CREATE TABLE IF NOT EXISTS archives (id TEXT PRIMARY KEY, digest TEXT NOT NULL, revision INTEGER NOT NULL);
|
|
26
|
+
CREATE TABLE IF NOT EXISTS revisions (archive_id TEXT NOT NULL REFERENCES archives(id), revision INTEGER NOT NULL,
|
|
27
|
+
digest TEXT NOT NULL, snapshot TEXT NOT NULL, chunks TEXT NOT NULL, PRIMARY KEY(archive_id, revision));
|
|
28
|
+
CREATE TABLE IF NOT EXISTS receipts (archive_id TEXT NOT NULL REFERENCES archives(id), destination TEXT NOT NULL,
|
|
29
|
+
digest TEXT NOT NULL, PRIMARY KEY(archive_id, destination));
|
|
30
|
+
CREATE TABLE IF NOT EXISTS outbox (archive_id TEXT NOT NULL REFERENCES archives(id), destination TEXT NOT NULL,
|
|
31
|
+
pending TEXT NOT NULL, PRIMARY KEY(archive_id, destination));`);
|
|
32
|
+
this.db.query("INSERT OR IGNORE INTO settings VALUES ('sourceId', ?)").run(randomUUID());
|
|
33
|
+
this.sourceId = (
|
|
34
|
+
this.db.query("SELECT value FROM settings WHERE key='sourceId'").get() as { value: string }
|
|
35
|
+
).value;
|
|
36
|
+
}
|
|
37
|
+
capture(input: unknown) {
|
|
38
|
+
const snapshot = archiveSnapshotSchema.parse(input);
|
|
39
|
+
const id = archiveKey(snapshot);
|
|
40
|
+
const hash = snapshotDigest(snapshot);
|
|
41
|
+
return this.db.transaction(() => {
|
|
42
|
+
const old = this.read(id);
|
|
43
|
+
if (old?.digest === hash) return { id, digest: hash, revision: old.revision, changed: false };
|
|
44
|
+
if (old && old.snapshot.capturedAt > snapshot.capturedAt)
|
|
45
|
+
throw new Error('Older capture cannot replace newer archive');
|
|
46
|
+
const revision = (old?.revision ?? 0) + 1;
|
|
47
|
+
const chunks = chunkArchive(snapshot);
|
|
48
|
+
this.db
|
|
49
|
+
.query(
|
|
50
|
+
'INSERT INTO archives VALUES (?, ?, ?) ON CONFLICT(id) DO UPDATE SET digest=excluded.digest, revision=excluded.revision',
|
|
51
|
+
)
|
|
52
|
+
.run(id, hash, revision);
|
|
53
|
+
this.db
|
|
54
|
+
.query('INSERT INTO revisions VALUES (?, ?, ?, ?, ?)')
|
|
55
|
+
.run(id, revision, hash, JSON.stringify(snapshot), JSON.stringify(chunks));
|
|
56
|
+
return { id, digest: hash, revision, changed: true };
|
|
57
|
+
})();
|
|
58
|
+
}
|
|
59
|
+
read(
|
|
60
|
+
id: string,
|
|
61
|
+
revision?: number,
|
|
62
|
+
):
|
|
63
|
+
| {
|
|
64
|
+
id: string;
|
|
65
|
+
revision: number;
|
|
66
|
+
digest: string;
|
|
67
|
+
snapshot: ArchiveSnapshot;
|
|
68
|
+
chunks: ArchiveChunk[];
|
|
69
|
+
}
|
|
70
|
+
| undefined {
|
|
71
|
+
const row = this.db
|
|
72
|
+
.query(`SELECT r.* FROM revisions r JOIN archives a ON a.id=r.archive_id
|
|
73
|
+
WHERE a.id=? AND r.revision=COALESCE(?, a.revision)`)
|
|
74
|
+
.get(id, revision ?? null) as any;
|
|
75
|
+
return row
|
|
76
|
+
? {
|
|
77
|
+
id,
|
|
78
|
+
revision: row.revision,
|
|
79
|
+
digest: row.digest,
|
|
80
|
+
snapshot: archiveSnapshotSchema.parse(JSON.parse(row.snapshot)),
|
|
81
|
+
chunks: JSON.parse(row.chunks),
|
|
82
|
+
}
|
|
83
|
+
: undefined;
|
|
84
|
+
}
|
|
85
|
+
list() {
|
|
86
|
+
return (
|
|
87
|
+
this.db.query('SELECT id FROM archives ORDER BY rowid DESC').all() as { id: string }[]
|
|
88
|
+
).map(({ id }) => {
|
|
89
|
+
const archive = this.read(id)!;
|
|
90
|
+
const { entries, ...snapshot } = archive.snapshot;
|
|
91
|
+
return {
|
|
92
|
+
id,
|
|
93
|
+
revision: archive.revision,
|
|
94
|
+
digest: archive.digest,
|
|
95
|
+
...snapshot,
|
|
96
|
+
entries: entries.length,
|
|
97
|
+
suggestedTags: categorySuggestions(archive.snapshot),
|
|
98
|
+
};
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
search(ids: string[], query: string, budget = 2048) {
|
|
102
|
+
return ids.map((id) => {
|
|
103
|
+
const archive = this.read(id);
|
|
104
|
+
if (!archive) throw new Error('Archive unavailable');
|
|
105
|
+
return {
|
|
106
|
+
id,
|
|
107
|
+
revision: archive.revision,
|
|
108
|
+
digest: archive.digest,
|
|
109
|
+
...selectChunks(archive.chunks, query, budget),
|
|
110
|
+
};
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
receipt(id: string, destination: string): string | null {
|
|
114
|
+
return (
|
|
115
|
+
(
|
|
116
|
+
this.db
|
|
117
|
+
.query('SELECT digest FROM receipts WHERE archive_id=? AND destination=?')
|
|
118
|
+
.get(id, destination) as { digest: string } | null
|
|
119
|
+
)?.digest ?? null
|
|
120
|
+
);
|
|
121
|
+
}
|
|
122
|
+
acknowledge(id: string, destination: string, hash: string) {
|
|
123
|
+
this.db
|
|
124
|
+
.query(
|
|
125
|
+
'INSERT INTO receipts VALUES (?, ?, ?) ON CONFLICT(archive_id,destination) DO UPDATE SET digest=excluded.digest',
|
|
126
|
+
)
|
|
127
|
+
.run(id, destination, hash);
|
|
128
|
+
}
|
|
129
|
+
pending(id: string, destination: string): { revision: number; keepIds: string[] } | null {
|
|
130
|
+
const row = this.db
|
|
131
|
+
.query('SELECT pending FROM outbox WHERE archive_id=? AND destination=?')
|
|
132
|
+
.get(id, destination) as { pending: string } | null;
|
|
133
|
+
return row ? JSON.parse(row.pending) : null;
|
|
134
|
+
}
|
|
135
|
+
enqueue(id: string, destination: string, pending: { revision: number; keepIds: string[] }) {
|
|
136
|
+
this.db
|
|
137
|
+
.query('INSERT OR IGNORE INTO outbox VALUES (?, ?, ?)')
|
|
138
|
+
.run(id, destination, JSON.stringify(pending));
|
|
139
|
+
}
|
|
140
|
+
delivered(id: string, destination: string, hash: string, placement: string) {
|
|
141
|
+
this.db.transaction(() => {
|
|
142
|
+
this.acknowledge(id, destination, hash);
|
|
143
|
+
this.acknowledge(id, `${destination}:keeps`, placement);
|
|
144
|
+
this.db.query('DELETE FROM outbox WHERE archive_id=? AND destination=?').run(id, destination);
|
|
145
|
+
})();
|
|
146
|
+
}
|
|
147
|
+
close() {
|
|
148
|
+
this.db.close();
|
|
149
|
+
}
|
|
150
|
+
}
|
package/src/preview.ts
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { lstatSync, readdirSync } from 'node:fs';
|
|
2
|
+
import { join, resolve } from 'node:path';
|
|
3
|
+
import { importTranscriptFile } from './import-file';
|
|
4
|
+
|
|
5
|
+
/** Inspect explicit source paths without opening an archive store or publishing data. */
|
|
6
|
+
export function previewImports(path: string, source: 'codex' | 'claude-code', limit = 100) {
|
|
7
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > 1000)
|
|
8
|
+
throw new Error('Preview limit must be between 1 and 1000');
|
|
9
|
+
const files: string[] = [];
|
|
10
|
+
let truncated = false;
|
|
11
|
+
const visit = (candidate: string) => {
|
|
12
|
+
if (files.length > limit) {
|
|
13
|
+
truncated = true;
|
|
14
|
+
return;
|
|
15
|
+
}
|
|
16
|
+
const stat = lstatSync(candidate);
|
|
17
|
+
if (stat.isSymbolicLink()) return;
|
|
18
|
+
if (stat.isDirectory()) {
|
|
19
|
+
for (const entry of readdirSync(candidate).sort()) {
|
|
20
|
+
visit(join(candidate, entry));
|
|
21
|
+
if (files.length > limit) break;
|
|
22
|
+
}
|
|
23
|
+
} else if (stat.isFile() && candidate.endsWith('.jsonl')) files.push(candidate);
|
|
24
|
+
};
|
|
25
|
+
visit(resolve(path));
|
|
26
|
+
truncated ||= files.length > limit;
|
|
27
|
+
const sessions = files.slice(0, limit).map((file) => {
|
|
28
|
+
try {
|
|
29
|
+
const snapshot = importTranscriptFile(file, {
|
|
30
|
+
source,
|
|
31
|
+
sourceId: '00000000-0000-4000-8000-000000000000',
|
|
32
|
+
});
|
|
33
|
+
return {
|
|
34
|
+
file,
|
|
35
|
+
status: 'ready' as const,
|
|
36
|
+
source,
|
|
37
|
+
sessionId: snapshot.sessionId,
|
|
38
|
+
title: snapshot.title,
|
|
39
|
+
entries: snapshot.entries.length,
|
|
40
|
+
characters: snapshot.entries.reduce((n, entry) => n + entry.text.length, 0),
|
|
41
|
+
coverage: snapshot.coverage,
|
|
42
|
+
};
|
|
43
|
+
} catch (error) {
|
|
44
|
+
return {
|
|
45
|
+
file,
|
|
46
|
+
status: 'rejected' as const,
|
|
47
|
+
error: error instanceof Error ? error.message : 'Unable to inspect transcript',
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
});
|
|
51
|
+
return {
|
|
52
|
+
sessions,
|
|
53
|
+
truncated,
|
|
54
|
+
ready: sessions.filter((session) => session.status === 'ready').length,
|
|
55
|
+
rejected: sessions.filter((session) => session.status === 'rejected').length,
|
|
56
|
+
};
|
|
57
|
+
}
|