@indigoai-us/hq-cloud 6.18.71 → 6.18.72
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +126 -16
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +68 -0
- package/dist/cli/sync.js.map +1 -1
- package/dist/lib/append-only-merge.d.ts +21 -0
- package/dist/lib/append-only-merge.d.ts.map +1 -0
- package/dist/lib/append-only-merge.js +107 -0
- package/dist/lib/append-only-merge.js.map +1 -0
- package/package.json +1 -1
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/** True for a path whose body is append-only lines (`*.jsonl`, any depth). */
|
|
2
|
+
export declare function isAppendOnlyPath(relativePath: string): boolean;
|
|
3
|
+
/**
|
|
4
|
+
* Merge two append-only bodies: the local body, then the cloud lines it lacks.
|
|
5
|
+
* The result always begins with the local body's exact bytes. Returns the
|
|
6
|
+
* local body itself when the cloud has no line it lacks, the cloud body itself
|
|
7
|
+
* when the merge equals it, and null when either body is not UTF-8 text.
|
|
8
|
+
*/
|
|
9
|
+
export declare function mergeAppendOnly(localBody: Buffer, remoteBody: Buffer): Buffer | null;
|
|
10
|
+
/**
|
|
11
|
+
* Append the merge's new bytes to `file`, but only while the file still holds
|
|
12
|
+
* exactly `expected`, the body the merge read. The file is opened for append
|
|
13
|
+
* and never replaced, so a writer holding it open keeps writing to it.
|
|
14
|
+
*
|
|
15
|
+
* Returns "appended" when the file now ends with the merge (or already equals
|
|
16
|
+
* it), and "changed" when another writer changed the file after the merge read
|
|
17
|
+
* it; the next pass then merges again from the new body. Filesystem failures
|
|
18
|
+
* are thrown so the caller can report them.
|
|
19
|
+
*/
|
|
20
|
+
export declare function appendIfUnchanged(file: string, expected: Buffer, merged: Buffer): "appended" | "changed";
|
|
21
|
+
//# sourceMappingURL=append-only-merge.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"append-only-merge.d.ts","sourceRoot":"","sources":["../../src/lib/append-only-merge.ts"],"names":[],"mappings":"AAmBA,8EAA8E;AAC9E,wBAAgB,gBAAgB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAG9D;AA+BD;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CA0BpF;AAED;;;;;;;;;GASG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS,CASxG"}
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Line merge for append-only files.
|
|
3
|
+
*
|
|
4
|
+
* Several machines append to the same `.jsonl` files (the handoff index, event
|
|
5
|
+
* logs). When two of them append between syncs, both bodies changed since the
|
|
6
|
+
* journal and sync used to park one of them. Their lines can always be combined
|
|
7
|
+
* without loss, so a conflict on such a file is merged instead.
|
|
8
|
+
*
|
|
9
|
+
* The merge keeps the local body exactly as it is and adds the cloud lines it
|
|
10
|
+
* lacks after it, in cloud order. The local file is therefore only ever
|
|
11
|
+
* appended to, never replaced: a program that holds the file open for append
|
|
12
|
+
* keeps writing to the same file, and nothing it writes is lost.
|
|
13
|
+
*
|
|
14
|
+
* Machines converge through the push fence: a merged body holds every line of
|
|
15
|
+
* both sides, it is published under an If-Match on the cloud version it was
|
|
16
|
+
* merged against, and a machine with no new lines of its own simply pulls.
|
|
17
|
+
*/
|
|
18
|
+
import * as fs from "node:fs";
|
|
19
|
+
/** True for a path whose body is append-only lines (`*.jsonl`, any depth). */
|
|
20
|
+
export function isAppendOnlyPath(relativePath) {
|
|
21
|
+
const base = relativePath.split(/[\\/]/).pop() ?? "";
|
|
22
|
+
return base.length > ".jsonl".length && base.toLowerCase().endsWith(".jsonl");
|
|
23
|
+
}
|
|
24
|
+
function splitLines(body) {
|
|
25
|
+
const lines = [];
|
|
26
|
+
const pattern = /([^\r\n]*)(\r\n|\n|\r)?/g;
|
|
27
|
+
let match;
|
|
28
|
+
while ((match = pattern.exec(body)) !== null) {
|
|
29
|
+
if (match[0] === "")
|
|
30
|
+
break;
|
|
31
|
+
lines.push({ text: match[1], eol: match[2] ?? "" });
|
|
32
|
+
}
|
|
33
|
+
return lines;
|
|
34
|
+
}
|
|
35
|
+
function decodeText(body) {
|
|
36
|
+
if (body.includes(0))
|
|
37
|
+
return null;
|
|
38
|
+
try {
|
|
39
|
+
return new TextDecoder("utf-8", { fatal: true }).decode(body);
|
|
40
|
+
}
|
|
41
|
+
catch {
|
|
42
|
+
return null;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
function defaultEol(lines) {
|
|
46
|
+
return lines.find((line) => line.eol !== "")?.eol ?? "\n";
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Merge two append-only bodies: the local body, then the cloud lines it lacks.
|
|
50
|
+
* The result always begins with the local body's exact bytes. Returns the
|
|
51
|
+
* local body itself when the cloud has no line it lacks, the cloud body itself
|
|
52
|
+
* when the merge equals it, and null when either body is not UTF-8 text.
|
|
53
|
+
*/
|
|
54
|
+
export function mergeAppendOnly(localBody, remoteBody) {
|
|
55
|
+
const localText = decodeText(localBody);
|
|
56
|
+
const remoteText = decodeText(remoteBody);
|
|
57
|
+
if (localText === null || remoteText === null)
|
|
58
|
+
return null;
|
|
59
|
+
const localLines = splitLines(localText);
|
|
60
|
+
const remoteLines = splitLines(remoteText);
|
|
61
|
+
const remaining = new Map();
|
|
62
|
+
for (const line of localLines)
|
|
63
|
+
remaining.set(line.text, (remaining.get(line.text) ?? 0) + 1);
|
|
64
|
+
const remoteOnly = [];
|
|
65
|
+
for (const line of remoteLines) {
|
|
66
|
+
const count = remaining.get(line.text) ?? 0;
|
|
67
|
+
if (count > 0)
|
|
68
|
+
remaining.set(line.text, count - 1);
|
|
69
|
+
else
|
|
70
|
+
remoteOnly.push(line);
|
|
71
|
+
}
|
|
72
|
+
if (remoteOnly.length === 0)
|
|
73
|
+
return localBody;
|
|
74
|
+
const localEol = defaultEol(localLines);
|
|
75
|
+
const remoteEol = defaultEol(remoteLines);
|
|
76
|
+
// Terminate an unterminated final local line, then add the cloud lines.
|
|
77
|
+
const lastLocal = localLines[localLines.length - 1];
|
|
78
|
+
let suffix = lastLocal && lastLocal.eol === "" ? localEol : "";
|
|
79
|
+
for (const line of remoteOnly)
|
|
80
|
+
suffix += line.text + (line.eol || remoteEol);
|
|
81
|
+
const merged = Buffer.concat([localBody, Buffer.from(suffix, "utf8")]);
|
|
82
|
+
return merged.equals(remoteBody) ? remoteBody : merged;
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Append the merge's new bytes to `file`, but only while the file still holds
|
|
86
|
+
* exactly `expected`, the body the merge read. The file is opened for append
|
|
87
|
+
* and never replaced, so a writer holding it open keeps writing to it.
|
|
88
|
+
*
|
|
89
|
+
* Returns "appended" when the file now ends with the merge (or already equals
|
|
90
|
+
* it), and "changed" when another writer changed the file after the merge read
|
|
91
|
+
* it; the next pass then merges again from the new body. Filesystem failures
|
|
92
|
+
* are thrown so the caller can report them.
|
|
93
|
+
*/
|
|
94
|
+
export function appendIfUnchanged(file, expected, merged) {
|
|
95
|
+
if (merged.length < expected.length || !merged.subarray(0, expected.length).equals(expected)) {
|
|
96
|
+
throw new Error("append-only merge does not extend the local body");
|
|
97
|
+
}
|
|
98
|
+
if (!fs.lstatSync(file).isFile())
|
|
99
|
+
return "changed";
|
|
100
|
+
if (!fs.readFileSync(file).equals(expected))
|
|
101
|
+
return "changed";
|
|
102
|
+
if (merged.length === expected.length)
|
|
103
|
+
return "appended";
|
|
104
|
+
fs.appendFileSync(file, merged.subarray(expected.length));
|
|
105
|
+
return "appended";
|
|
106
|
+
}
|
|
107
|
+
//# sourceMappingURL=append-only-merge.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"append-only-merge.js","sourceRoot":"","sources":["../../src/lib/append-only-merge.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AACH,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAE9B,8EAA8E;AAC9E,MAAM,UAAU,gBAAgB,CAAC,YAAoB;IACnD,MAAM,IAAI,GAAG,YAAY,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;IACrD,OAAO,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;AAChF,CAAC;AAOD,SAAS,UAAU,CAAC,IAAY;IAC9B,MAAM,KAAK,GAAW,EAAE,CAAC;IACzB,MAAM,OAAO,GAAG,0BAA0B,CAAC;IAC3C,IAAI,KAA6B,CAAC;IAClC,OAAO,CAAC,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QAC7C,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,EAAE;YAAE,MAAM;QAC3B,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAE,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IACvD,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,UAAU,CAAC,IAAY;IAC9B,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAClC,IAAI,CAAC;QACH,OAAO,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAChE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CAAC,KAAa;IAC/B,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,GAAG,IAAI,IAAI,CAAC;AAC5D,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,eAAe,CAAC,SAAiB,EAAE,UAAkB;IACnE,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC;IACxC,MAAM,UAAU,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;IAC1C,IAAI,SAAS,KAAK,IAAI,IAAI,UAAU,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IAE3D,MAAM,UAAU,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC;IACzC,MAAM,WAAW,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;IAC3C,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC5C,KAAK,MAAM,IAAI,IAAI,UAAU;QAAE,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAE7F,MAAM,UAAU,GAAW,EAAE,CAAC;IAC9B,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5C,IAAI,KAAK,GAAG,CAAC;YAAE,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;;YAC9C,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;IAED,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAC9C,MAAM,QAAQ,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;IACxC,MAAM,SAAS,GAAG,UAAU,CAAC,WAAW,CAAC,CAAC;IAC1C,wEAAwE;IACxE,MAAM,SAAS,GAAG,UAAU,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACpD,IAAI,MAAM,GAAG,SAAS,IAAI,SAAS,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;IAC/D,KAAK,MAAM,IAAI,IAAI,UAAU;QAAE,MAAM,IAAI,IAAI,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,IAAI,SAAS,CAAC,CAAC;IAC7E,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;IACvE,OAAO,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC;AACzD,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAY,EAAE,QAAgB,EAAE,MAAc;IAC9E,IAAI,MAAM,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC7F,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;IACtE,CAAC;IACD,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE;QAAE,OAAO,SAAS,CAAC;IACnD,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC;QAAE,OAAO,SAAS,CAAC;IAC9D,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM;QAAE,OAAO,UAAU,CAAC;IACzD,EAAE,CAAC,cAAc,CAAC,IAAI,EAAE,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;IAC1D,OAAO,UAAU,CAAC;AACpB,CAAC"}
|