@remnic/core 1.1.29 → 1.1.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/access-cli.js +13 -13
- package/dist/access-http.d.ts +2 -1
- package/dist/access-http.js +8 -8
- package/dist/access-mcp.d.ts +1 -1
- package/dist/access-mcp.js +7 -7
- package/dist/access-schema.d.ts +55 -5
- package/dist/access-schema.js +4 -2
- package/dist/{access-service-CEyV8XJ5.d.ts → access-service-CkZyb35d.d.ts} +10 -2
- package/dist/access-service.d.ts +1 -1
- package/dist/access-service.js +5 -5
- package/dist/briefing.js +2 -2
- package/dist/causal-consolidation.js +3 -3
- package/dist/{chunk-25YQM6XW.js → chunk-2IWUMAES.js} +3 -3
- package/dist/{chunk-6CB4E7ZV.js → chunk-3ZLVGM76.js} +4 -4
- package/dist/{chunk-QYHQ2JHL.js → chunk-43PJZYGL.js} +2 -2
- package/dist/{chunk-YITUHONZ.js → chunk-4KGVTPGD.js} +2 -2
- package/dist/{chunk-TR4DK5OH.js → chunk-76FLAAUC.js} +2 -2
- package/dist/{chunk-6BFAEWQS.js → chunk-77H5NU3M.js} +2 -2
- package/dist/{chunk-IANK6Y5W.js → chunk-A6KTB5R6.js} +2 -2
- package/dist/{chunk-7D6O46PF.js → chunk-BVF3AGJP.js} +2 -2
- package/dist/{chunk-4H6DURG6.js → chunk-JA3AK3PT.js} +2 -2
- package/dist/{chunk-RCZRL5BE.js → chunk-MRILGULB.js} +2 -2
- package/dist/{chunk-CWWDIQZB.js → chunk-QLLBRHAT.js} +8 -8
- package/dist/{chunk-2WIPXV3Y.js → chunk-RR2PKP3I.js} +2 -2
- package/dist/{chunk-3F24QTRI.js → chunk-SAZS2QZB.js} +2 -2
- package/dist/{chunk-VYU7PXUS.js → chunk-SIC6U3GZ.js} +2 -2
- package/dist/{chunk-WDSIV3AK.js → chunk-TPU5L5EY.js} +12 -12
- package/dist/{chunk-AMVN77EU.js → chunk-U7EJOMFC.js} +371 -91
- package/dist/chunk-U7EJOMFC.js.map +1 -0
- package/dist/{chunk-F33CJ5CH.js → chunk-VBJ7V5SK.js} +40 -8
- package/dist/chunk-VBJ7V5SK.js.map +1 -0
- package/dist/{chunk-6WV2HYTZ.js → chunk-W6AQJ2PY.js} +4 -4
- package/dist/{chunk-PUXCIHRL.js → chunk-XSZEP4SF.js} +2 -2
- package/dist/{chunk-NW7JW5GA.js → chunk-YROHKYBY.js} +41 -6
- package/dist/chunk-YROHKYBY.js.map +1 -0
- package/dist/{chunk-JUYT2J3K.js → chunk-YU5KIWYQ.js} +136 -8
- package/dist/chunk-YU5KIWYQ.js.map +1 -0
- package/dist/{chunk-LCTP7YRU.js → chunk-ZAVUCJ4H.js} +38 -7
- package/dist/chunk-ZAVUCJ4H.js.map +1 -0
- package/dist/{cli-BguVmIwO.d.ts → cli-kuh9PwZ5.d.ts} +1 -1
- package/dist/cli.d.ts +2 -2
- package/dist/cli.js +17 -17
- package/dist/compounding/engine.js +2 -2
- package/dist/connectors/codex-materialize-runner.js +2 -2
- package/dist/connectors/index.js +2 -2
- package/dist/entity-retrieval.js +2 -2
- package/dist/index.d.ts +4 -4
- package/dist/index.js +34 -22
- package/dist/index.js.map +1 -1
- package/dist/maintenance/memory-governance.js +2 -2
- package/dist/maintenance/rebuild-memory-lifecycle-ledger.js +2 -2
- package/dist/maintenance/rebuild-memory-projection.js +3 -3
- package/dist/mcp-memory-inspector-app.d.ts +1 -1
- package/dist/namespaces/migrate.js +3 -3
- package/dist/namespaces/storage.js +2 -2
- package/dist/offline-sync.d.ts +56 -1
- package/dist/offline-sync.js +15 -1
- package/dist/operator-toolkit.js +5 -5
- package/dist/orchestrator.js +9 -9
- package/dist/schemas.d.ts +22 -22
- package/dist/semantic-consolidation.js +3 -3
- package/dist/semantic-rule-promotion.js +2 -2
- package/dist/semantic-rule-verifier.js +2 -2
- package/dist/storage.d.ts +5 -0
- package/dist/storage.js +1 -1
- package/dist/transfer/types.d.ts +12 -12
- package/dist/verified-recall.js +2 -2
- package/package.json +1 -1
- package/src/access-http.test.ts +355 -0
- package/src/access-http.ts +149 -1
- package/src/access-schema.ts +58 -3
- package/src/access-service-namespace.test.ts +56 -1
- package/src/access-service-offline-file-content.test.ts +17 -0
- package/src/access-service.ts +47 -1
- package/src/index.ts +7 -0
- package/src/offline-sync.test.ts +1055 -1
- package/src/offline-sync.ts +465 -97
- package/src/storage.ts +36 -2
- package/dist/chunk-AMVN77EU.js.map +0 -1
- package/dist/chunk-F33CJ5CH.js.map +0 -1
- package/dist/chunk-JUYT2J3K.js.map +0 -1
- package/dist/chunk-LCTP7YRU.js.map +0 -1
- package/dist/chunk-NW7JW5GA.js.map +0 -1
- /package/dist/{chunk-25YQM6XW.js.map → chunk-2IWUMAES.js.map} +0 -0
- /package/dist/{chunk-6CB4E7ZV.js.map → chunk-3ZLVGM76.js.map} +0 -0
- /package/dist/{chunk-QYHQ2JHL.js.map → chunk-43PJZYGL.js.map} +0 -0
- /package/dist/{chunk-YITUHONZ.js.map → chunk-4KGVTPGD.js.map} +0 -0
- /package/dist/{chunk-TR4DK5OH.js.map → chunk-76FLAAUC.js.map} +0 -0
- /package/dist/{chunk-6BFAEWQS.js.map → chunk-77H5NU3M.js.map} +0 -0
- /package/dist/{chunk-IANK6Y5W.js.map → chunk-A6KTB5R6.js.map} +0 -0
- /package/dist/{chunk-7D6O46PF.js.map → chunk-BVF3AGJP.js.map} +0 -0
- /package/dist/{chunk-4H6DURG6.js.map → chunk-JA3AK3PT.js.map} +0 -0
- /package/dist/{chunk-RCZRL5BE.js.map → chunk-MRILGULB.js.map} +0 -0
- /package/dist/{chunk-CWWDIQZB.js.map → chunk-QLLBRHAT.js.map} +0 -0
- /package/dist/{chunk-2WIPXV3Y.js.map → chunk-RR2PKP3I.js.map} +0 -0
- /package/dist/{chunk-3F24QTRI.js.map → chunk-SAZS2QZB.js.map} +0 -0
- /package/dist/{chunk-VYU7PXUS.js.map → chunk-SIC6U3GZ.js.map} +0 -0
- /package/dist/{chunk-WDSIV3AK.js.map → chunk-TPU5L5EY.js.map} +0 -0
- /package/dist/{chunk-6WV2HYTZ.js.map → chunk-W6AQJ2PY.js.map} +0 -0
- /package/dist/{chunk-PUXCIHRL.js.map → chunk-XSZEP4SF.js.map} +0 -0
package/src/access-http.ts
CHANGED
|
@@ -5,6 +5,7 @@ import { existsSync } from "node:fs";
|
|
|
5
5
|
import { readFile } from "node:fs/promises";
|
|
6
6
|
import path from "node:path";
|
|
7
7
|
import { fileURLToPath, URL } from "node:url";
|
|
8
|
+
import { gunzipSync } from "node:zlib";
|
|
8
9
|
import { log } from "./logger.js";
|
|
9
10
|
import { EngramAccessInputError, type EngramAccessService } from "./access-service.js";
|
|
10
11
|
import { EngramMcpServer } from "./access-mcp.js";
|
|
@@ -12,6 +13,7 @@ import { validateRequest, type SchemaName, type SchemaTypeFor } from "./access-s
|
|
|
12
13
|
import {
|
|
13
14
|
OFFLINE_SYNC_APPLY_MAX_BODY_BYTES,
|
|
14
15
|
OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES,
|
|
16
|
+
OFFLINE_SYNC_SNAPSHOT_BASE_MAX_BODY_BYTES,
|
|
15
17
|
} from "./offline-sync.js";
|
|
16
18
|
import type { RecallDisclosure, RecallPlanMode } from "./types.js";
|
|
17
19
|
import { isRecallDisclosure } from "./types.js";
|
|
@@ -125,6 +127,16 @@ function parseTrustZoneFilter(raw: string | null): TrustZoneName | undefined {
|
|
|
125
127
|
throw new HttpError(400, "zone must be one of quarantine|working|trusted", "invalid_zone_filter");
|
|
126
128
|
}
|
|
127
129
|
|
|
130
|
+
function summarizeHttpRequest(req: IncomingMessage): string {
|
|
131
|
+
const method = req.method ?? "UNKNOWN";
|
|
132
|
+
try {
|
|
133
|
+
const parsed = new URL(req.url ?? "/", "http://localhost");
|
|
134
|
+
return `${method} ${parsed.pathname}`;
|
|
135
|
+
} catch {
|
|
136
|
+
return `${method} ${(req.url ?? "/").split("?")[0]}`;
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
128
140
|
/**
|
|
129
141
|
* Decode a `:peerId` URL path segment, converting malformed percent-encoded
|
|
130
142
|
* input (e.g., `%E0%A4%A`) into a 400 client error rather than letting
|
|
@@ -217,6 +229,10 @@ export class EngramAccessHttpServer {
|
|
|
217
229
|
res.destroy(err as Error);
|
|
218
230
|
return;
|
|
219
231
|
}
|
|
232
|
+
log.error(
|
|
233
|
+
`engram access HTTP internal error [${correlationId}] ${summarizeHttpRequest(req)}`,
|
|
234
|
+
err,
|
|
235
|
+
);
|
|
220
236
|
this.respondJson(res, 500, { error: "internal_error", code: "internal_error" });
|
|
221
237
|
});
|
|
222
238
|
});
|
|
@@ -630,6 +646,63 @@ export class EngramAccessHttpServer {
|
|
|
630
646
|
return;
|
|
631
647
|
}
|
|
632
648
|
|
|
649
|
+
if (
|
|
650
|
+
req.method === "GET" &&
|
|
651
|
+
(pathname === "/engram/v1/offline-sync/snapshot-stream" ||
|
|
652
|
+
pathname === "/remnic/v1/offline-sync/snapshot-stream")
|
|
653
|
+
) {
|
|
654
|
+
const includeTranscriptsRaw = parsed.searchParams.get("include_transcripts");
|
|
655
|
+
const includeContentRaw = parsed.searchParams.get("content");
|
|
656
|
+
if (
|
|
657
|
+
includeTranscriptsRaw !== null &&
|
|
658
|
+
includeTranscriptsRaw !== "true" &&
|
|
659
|
+
includeTranscriptsRaw !== "false"
|
|
660
|
+
) {
|
|
661
|
+
throw new EngramAccessInputError(
|
|
662
|
+
`include_transcripts must be one of: true, false (got: ${includeTranscriptsRaw})`,
|
|
663
|
+
);
|
|
664
|
+
}
|
|
665
|
+
if (
|
|
666
|
+
includeContentRaw !== null &&
|
|
667
|
+
includeContentRaw !== "false"
|
|
668
|
+
) {
|
|
669
|
+
throw new EngramAccessInputError("snapshot-stream content must be false");
|
|
670
|
+
}
|
|
671
|
+
const namespaceParam = parsed.searchParams.get("namespace");
|
|
672
|
+
const result = await this.service.offlineSyncSnapshotStream({
|
|
673
|
+
namespace: this.resolveNamespace(
|
|
674
|
+
req,
|
|
675
|
+
namespaceParam && namespaceParam.length > 0 ? namespaceParam : undefined,
|
|
676
|
+
),
|
|
677
|
+
principal: this.resolveRequestPrincipal(req),
|
|
678
|
+
includeTranscripts: includeTranscriptsRaw !== "false",
|
|
679
|
+
includeContent: false,
|
|
680
|
+
});
|
|
681
|
+
await this.respondOfflineSnapshotStream(res, result);
|
|
682
|
+
return;
|
|
683
|
+
}
|
|
684
|
+
|
|
685
|
+
if (
|
|
686
|
+
req.method === "POST" &&
|
|
687
|
+
(pathname === "/engram/v1/offline-sync/snapshot" || pathname === "/remnic/v1/offline-sync/snapshot")
|
|
688
|
+
) {
|
|
689
|
+
const body = await this.readValidatedBody(
|
|
690
|
+
req,
|
|
691
|
+
"offlineSyncSnapshot",
|
|
692
|
+
OFFLINE_SYNC_SNAPSHOT_BASE_MAX_BODY_BYTES,
|
|
693
|
+
);
|
|
694
|
+
const result = await this.service.offlineSyncSnapshot({
|
|
695
|
+
namespace: this.resolveNamespace(req, body.namespace),
|
|
696
|
+
principal: this.resolveRequestPrincipal(req),
|
|
697
|
+
includeTranscripts: body.includeTranscripts,
|
|
698
|
+
includeContent: body.includeContent,
|
|
699
|
+
baseFiles: body.baseFiles,
|
|
700
|
+
...(body.baseCapturedAt ? { baseCapturedAt: new Date(body.baseCapturedAt) } : {}),
|
|
701
|
+
});
|
|
702
|
+
this.respondJson(res, 200, result);
|
|
703
|
+
return;
|
|
704
|
+
}
|
|
705
|
+
|
|
633
706
|
if (
|
|
634
707
|
req.method === "POST" &&
|
|
635
708
|
(pathname === "/engram/v1/offline-sync/files" || pathname === "/remnic/v1/offline-sync/files")
|
|
@@ -717,6 +790,7 @@ export class EngramAccessHttpServer {
|
|
|
717
790
|
namespace: this.resolveNamespace(req, body.namespace),
|
|
718
791
|
principal: this.resolveRequestPrincipal(req),
|
|
719
792
|
changeset: body.changeset,
|
|
793
|
+
returnCurrentFiles: body.returnCurrentFiles,
|
|
720
794
|
});
|
|
721
795
|
this.respondJson(res, 200, result);
|
|
722
796
|
return;
|
|
@@ -1868,6 +1942,62 @@ export class EngramAccessHttpServer {
|
|
|
1868
1942
|
res.end(body);
|
|
1869
1943
|
}
|
|
1870
1944
|
|
|
1945
|
+
private async respondOfflineSnapshotStream(
|
|
1946
|
+
res: ServerResponse,
|
|
1947
|
+
snapshot: Awaited<ReturnType<EngramAccessService["offlineSyncSnapshotStream"]>>,
|
|
1948
|
+
): Promise<void> {
|
|
1949
|
+
res.statusCode = 200;
|
|
1950
|
+
res.setHeader("content-type", "application/x-ndjson; charset=utf-8");
|
|
1951
|
+
res.setHeader("cache-control", "no-store");
|
|
1952
|
+
const cid = correlationIdStore.getStore();
|
|
1953
|
+
if (cid) {
|
|
1954
|
+
res.setHeader("x-request-id", cid);
|
|
1955
|
+
}
|
|
1956
|
+
const waitForDrainOrClose = async (): Promise<boolean> => new Promise((resolve, reject) => {
|
|
1957
|
+
const cleanup = () => {
|
|
1958
|
+
res.off("drain", onDrain);
|
|
1959
|
+
res.off("close", onClose);
|
|
1960
|
+
res.off("error", onError);
|
|
1961
|
+
};
|
|
1962
|
+
const onDrain = () => {
|
|
1963
|
+
cleanup();
|
|
1964
|
+
resolve(true);
|
|
1965
|
+
};
|
|
1966
|
+
const onClose = () => {
|
|
1967
|
+
cleanup();
|
|
1968
|
+
resolve(false);
|
|
1969
|
+
};
|
|
1970
|
+
const onError = (error: Error) => {
|
|
1971
|
+
cleanup();
|
|
1972
|
+
reject(error);
|
|
1973
|
+
};
|
|
1974
|
+
res.once("drain", onDrain);
|
|
1975
|
+
res.once("close", onClose);
|
|
1976
|
+
res.once("error", onError);
|
|
1977
|
+
});
|
|
1978
|
+
const writeLine = async (payload: unknown): Promise<boolean> => {
|
|
1979
|
+
if (res.destroyed || res.writableEnded) return false;
|
|
1980
|
+
if (res.write(`${JSON.stringify(payload)}\n`)) return true;
|
|
1981
|
+
if (res.destroyed || res.writableEnded) return false;
|
|
1982
|
+
return waitForDrainOrClose();
|
|
1983
|
+
};
|
|
1984
|
+
if (!await writeLine({
|
|
1985
|
+
type: "snapshot",
|
|
1986
|
+
namespace: snapshot.namespace,
|
|
1987
|
+
format: snapshot.format,
|
|
1988
|
+
schemaVersion: snapshot.schemaVersion,
|
|
1989
|
+
createdAt: snapshot.createdAt,
|
|
1990
|
+
sourceId: snapshot.sourceId,
|
|
1991
|
+
includeTranscripts: snapshot.includeTranscripts,
|
|
1992
|
+
})) return;
|
|
1993
|
+
for await (const file of snapshot.files) {
|
|
1994
|
+
if (!await writeLine({ type: "file", file })) return;
|
|
1995
|
+
}
|
|
1996
|
+
if (!res.destroyed && !res.writableEnded) {
|
|
1997
|
+
res.end();
|
|
1998
|
+
}
|
|
1999
|
+
}
|
|
2000
|
+
|
|
1871
2001
|
private respondBinary(
|
|
1872
2002
|
res: ServerResponse,
|
|
1873
2003
|
status: number,
|
|
@@ -1926,6 +2056,10 @@ export class EngramAccessHttpServer {
|
|
|
1926
2056
|
req: IncomingMessage,
|
|
1927
2057
|
maxBodyBytes = this.maxBodyBytes,
|
|
1928
2058
|
): Promise<Record<string, unknown>> {
|
|
2059
|
+
const encoding = (this.readOptionalHeader(req, "content-encoding") ?? "identity").toLowerCase();
|
|
2060
|
+
if (encoding !== "identity" && encoding !== "gzip") {
|
|
2061
|
+
throw new HttpError(415, "unsupported_content_encoding", "unsupported_content_encoding");
|
|
2062
|
+
}
|
|
1929
2063
|
const chunks: Buffer[] = [];
|
|
1930
2064
|
let total = 0;
|
|
1931
2065
|
for await (const chunk of req) {
|
|
@@ -1937,7 +2071,21 @@ export class EngramAccessHttpServer {
|
|
|
1937
2071
|
chunks.push(buffer);
|
|
1938
2072
|
}
|
|
1939
2073
|
if (chunks.length === 0) return {};
|
|
1940
|
-
|
|
2074
|
+
let body = Buffer.concat(chunks, total);
|
|
2075
|
+
if (encoding === "gzip") {
|
|
2076
|
+
try {
|
|
2077
|
+
body = gunzipSync(body, { maxOutputLength: maxBodyBytes });
|
|
2078
|
+
} catch (error) {
|
|
2079
|
+
if ((error as NodeJS.ErrnoException).code === "ERR_BUFFER_TOO_LARGE") {
|
|
2080
|
+
throw new HttpError(413, "request_body_too_large", "request_body_too_large");
|
|
2081
|
+
}
|
|
2082
|
+
throw new HttpError(400, "invalid_gzip_body", "invalid_gzip_body");
|
|
2083
|
+
}
|
|
2084
|
+
if (body.byteLength > maxBodyBytes) {
|
|
2085
|
+
throw new HttpError(413, "request_body_too_large", "request_body_too_large");
|
|
2086
|
+
}
|
|
2087
|
+
}
|
|
2088
|
+
const raw = body.toString("utf-8").trim();
|
|
1941
2089
|
if (raw.length === 0) return {};
|
|
1942
2090
|
let parsed: unknown;
|
|
1943
2091
|
try {
|
package/src/access-schema.ts
CHANGED
|
@@ -9,8 +9,12 @@ import {
|
|
|
9
9
|
ACTION_CONFIDENCE_RULE_KINDS,
|
|
10
10
|
} from "./action-confidence.js";
|
|
11
11
|
import { isValidCapsuleSince } from "./transfer/capsule-export.js";
|
|
12
|
+
import { validateArchiveRelativePath } from "./transfer/fs-utils.js";
|
|
12
13
|
import { CAPSULE_ID_PATTERN } from "./transfer/types.js";
|
|
13
|
-
import {
|
|
14
|
+
import {
|
|
15
|
+
OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES,
|
|
16
|
+
OFFLINE_SYNC_MAX_MTIME_MS,
|
|
17
|
+
} from "./offline-sync.js";
|
|
14
18
|
|
|
15
19
|
// ---------------------------------------------------------------------------
|
|
16
20
|
// Error formatting
|
|
@@ -369,10 +373,57 @@ export const capsuleListRequestSchema = z
|
|
|
369
373
|
// Offline sync
|
|
370
374
|
// ---------------------------------------------------------------------------
|
|
371
375
|
|
|
376
|
+
function isValidOfflineSyncPath(value: string): boolean {
|
|
377
|
+
try {
|
|
378
|
+
validateArchiveRelativePath(value, "path");
|
|
379
|
+
return true;
|
|
380
|
+
} catch {
|
|
381
|
+
return false;
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
const offlineSyncPathSchema = z
|
|
386
|
+
.string()
|
|
387
|
+
.trim()
|
|
388
|
+
.min(1, "path must be non-empty")
|
|
389
|
+
.max(4096)
|
|
390
|
+
.refine(
|
|
391
|
+
isValidOfflineSyncPath,
|
|
392
|
+
"path must be a POSIX relative path without unsafe segments",
|
|
393
|
+
);
|
|
394
|
+
|
|
395
|
+
const offlineSyncFileStateSchema = z.object({
|
|
396
|
+
path: offlineSyncPathSchema,
|
|
397
|
+
sha256: z.string().regex(/^[a-f0-9]{64}$/i, "sha256 must be a 64-character hex digest"),
|
|
398
|
+
bytes: z.number().int().min(0),
|
|
399
|
+
mtimeMs: z.number().finite().min(0).max(OFFLINE_SYNC_MAX_MTIME_MS),
|
|
400
|
+
});
|
|
401
|
+
|
|
402
|
+
const offlineSyncBaseCapturedAtSchema = z
|
|
403
|
+
.string()
|
|
404
|
+
.trim()
|
|
405
|
+
.min(1, "baseCapturedAt must be non-empty when provided")
|
|
406
|
+
.max(64)
|
|
407
|
+
.refine((value) => Number.isFinite(Date.parse(value)), {
|
|
408
|
+
message: "baseCapturedAt must be a valid ISO 8601 timestamp",
|
|
409
|
+
});
|
|
410
|
+
|
|
411
|
+
export const offlineSyncSnapshotRequestSchema = z.object({
|
|
412
|
+
namespace: namespaceSchema,
|
|
413
|
+
includeTranscripts: z.boolean().optional(),
|
|
414
|
+
includeContent: z.boolean().optional(),
|
|
415
|
+
baseCapturedAt: offlineSyncBaseCapturedAtSchema.optional(),
|
|
416
|
+
baseFiles: z
|
|
417
|
+
.array(offlineSyncFileStateSchema)
|
|
418
|
+
.max(300_000, "baseFiles must contain 300000 or fewer entries")
|
|
419
|
+
.optional(),
|
|
420
|
+
});
|
|
421
|
+
|
|
372
422
|
export const offlineSyncApplyRequestSchema = z
|
|
373
423
|
.object({
|
|
374
424
|
namespace: namespaceSchema,
|
|
375
425
|
changeset: z.unknown(),
|
|
426
|
+
returnCurrentFiles: z.boolean().optional(),
|
|
376
427
|
})
|
|
377
428
|
.refine((value) => value.changeset !== undefined && value.changeset !== null, {
|
|
378
429
|
message: "changeset is required",
|
|
@@ -383,14 +434,14 @@ export const offlineSyncFilesRequestSchema = z.object({
|
|
|
383
434
|
namespace: namespaceSchema,
|
|
384
435
|
includeTranscripts: z.boolean().optional(),
|
|
385
436
|
paths: z
|
|
386
|
-
.array(
|
|
437
|
+
.array(offlineSyncPathSchema)
|
|
387
438
|
.max(5000, "paths must contain 5000 or fewer entries"),
|
|
388
439
|
});
|
|
389
440
|
|
|
390
441
|
export const offlineSyncFileContentRequestSchema = z.object({
|
|
391
442
|
namespace: namespaceSchema,
|
|
392
443
|
includeTranscripts: z.boolean().optional(),
|
|
393
|
-
path:
|
|
444
|
+
path: offlineSyncPathSchema,
|
|
394
445
|
offset: z.number().int().min(0).optional(),
|
|
395
446
|
length: z.number().int().min(1).max(OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES).optional(),
|
|
396
447
|
});
|
|
@@ -461,6 +512,7 @@ export type CapsuleExportRequest = z.infer<typeof capsuleExportRequestSchema>;
|
|
|
461
512
|
export type CapsuleImportRequest = z.infer<typeof capsuleImportRequestSchema>;
|
|
462
513
|
export type CapsuleListRequest = z.infer<typeof capsuleListRequestSchema>;
|
|
463
514
|
export type OfflineSyncApplyRequest = z.infer<typeof offlineSyncApplyRequestSchema>;
|
|
515
|
+
export type OfflineSyncSnapshotRequest = z.infer<typeof offlineSyncSnapshotRequestSchema>;
|
|
464
516
|
export type OfflineSyncFilesRequest = z.infer<typeof offlineSyncFilesRequestSchema>;
|
|
465
517
|
export type OfflineSyncFileContentRequest = z.infer<typeof offlineSyncFileContentRequestSchema>;
|
|
466
518
|
export type ActionConfidenceRequest = z.infer<typeof actionConfidenceRequestSchema>;
|
|
@@ -486,6 +538,7 @@ export type SchemaName =
|
|
|
486
538
|
| "capsuleExport"
|
|
487
539
|
| "capsuleImport"
|
|
488
540
|
| "capsuleList"
|
|
541
|
+
| "offlineSyncSnapshot"
|
|
489
542
|
| "offlineSyncFiles"
|
|
490
543
|
| "offlineSyncFileContent"
|
|
491
544
|
| "offlineSyncApply"
|
|
@@ -508,6 +561,7 @@ export type SchemaTypeFor<N extends SchemaName> =
|
|
|
508
561
|
: N extends "capsuleExport" ? CapsuleExportRequest
|
|
509
562
|
: N extends "capsuleImport" ? CapsuleImportRequest
|
|
510
563
|
: N extends "capsuleList" ? CapsuleListRequest
|
|
564
|
+
: N extends "offlineSyncSnapshot" ? OfflineSyncSnapshotRequest
|
|
511
565
|
: N extends "offlineSyncFiles" ? OfflineSyncFilesRequest
|
|
512
566
|
: N extends "offlineSyncFileContent" ? OfflineSyncFileContentRequest
|
|
513
567
|
: N extends "offlineSyncApply" ? OfflineSyncApplyRequest
|
|
@@ -531,6 +585,7 @@ const schemas: Record<SchemaName, z.ZodTypeAny> = {
|
|
|
531
585
|
capsuleExport: capsuleExportRequestSchema,
|
|
532
586
|
capsuleImport: capsuleImportRequestSchema,
|
|
533
587
|
capsuleList: capsuleListRequestSchema,
|
|
588
|
+
offlineSyncSnapshot: offlineSyncSnapshotRequestSchema,
|
|
534
589
|
offlineSyncFiles: offlineSyncFilesRequestSchema,
|
|
535
590
|
offlineSyncFileContent: offlineSyncFileContentRequestSchema,
|
|
536
591
|
offlineSyncApply: offlineSyncApplyRequestSchema,
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import assert from "node:assert/strict";
|
|
2
|
-
import {
|
|
2
|
+
import { createHash } from "node:crypto";
|
|
3
|
+
import { mkdir, mkdtemp, readFile, rm, symlink, utimes, writeFile } from "node:fs/promises";
|
|
3
4
|
import os from "node:os";
|
|
4
5
|
import path from "node:path";
|
|
5
6
|
import test from "node:test";
|
|
@@ -184,3 +185,57 @@ test("offlineSyncFiles reports symlink requested paths as input errors", async (
|
|
|
184
185
|
await rm(root, { recursive: true, force: true });
|
|
185
186
|
}
|
|
186
187
|
});
|
|
188
|
+
|
|
189
|
+
test("offlineSyncSnapshot ignores client capture time for server fast-base scans", async () => {
|
|
190
|
+
const root = await mkdtemp(path.join(os.tmpdir(), "remnic-offline-snapshot-client-clock-"));
|
|
191
|
+
try {
|
|
192
|
+
const relPath = "facts/a.md";
|
|
193
|
+
const filePath = path.join(root, relPath);
|
|
194
|
+
const oldContent = Buffer.from("alpha");
|
|
195
|
+
const newContent = Buffer.from("bravo");
|
|
196
|
+
const mtimeMs = 1_700_000_000_000;
|
|
197
|
+
await mkdir(path.dirname(filePath), { recursive: true });
|
|
198
|
+
await writeFile(filePath, oldContent);
|
|
199
|
+
await utimes(filePath, mtimeMs / 1000, mtimeMs / 1000);
|
|
200
|
+
const baseFile = {
|
|
201
|
+
path: relPath,
|
|
202
|
+
sha256: createHash("sha256").update(oldContent).digest("hex"),
|
|
203
|
+
bytes: oldContent.byteLength,
|
|
204
|
+
mtimeMs,
|
|
205
|
+
};
|
|
206
|
+
await writeFile(filePath, newContent);
|
|
207
|
+
await utimes(filePath, mtimeMs / 1000, mtimeMs / 1000);
|
|
208
|
+
|
|
209
|
+
const { service } = makeService();
|
|
210
|
+
(service as unknown as {
|
|
211
|
+
orchestrator: {
|
|
212
|
+
config: PluginConfig;
|
|
213
|
+
getStorage(namespace: string): Promise<StorageManager>;
|
|
214
|
+
};
|
|
215
|
+
}).orchestrator.getStorage = async () => ({
|
|
216
|
+
dir: root,
|
|
217
|
+
async readOfflineSyncFile(targetPath: string) {
|
|
218
|
+
return readFile(targetPath);
|
|
219
|
+
},
|
|
220
|
+
async digestOfflineSyncFile(targetPath: string) {
|
|
221
|
+
const content = await readFile(targetPath);
|
|
222
|
+
return {
|
|
223
|
+
sha256: createHash("sha256").update(content).digest("hex"),
|
|
224
|
+
bytes: content.byteLength,
|
|
225
|
+
};
|
|
226
|
+
},
|
|
227
|
+
} as unknown as StorageManager);
|
|
228
|
+
|
|
229
|
+
const snapshot = await service.offlineSyncSnapshot({
|
|
230
|
+
namespace: "team",
|
|
231
|
+
principal: "reader",
|
|
232
|
+
includeContent: false,
|
|
233
|
+
baseFiles: [baseFile],
|
|
234
|
+
baseCapturedAt: new Date(Date.now() + 60_000),
|
|
235
|
+
});
|
|
236
|
+
|
|
237
|
+
assert.equal(snapshot.files[0]?.sha256, createHash("sha256").update(newContent).digest("hex"));
|
|
238
|
+
} finally {
|
|
239
|
+
await rm(root, { recursive: true, force: true });
|
|
240
|
+
}
|
|
241
|
+
});
|
|
@@ -2,6 +2,7 @@ import assert from "node:assert/strict";
|
|
|
2
2
|
import test from "node:test";
|
|
3
3
|
|
|
4
4
|
import { EngramAccessInputError, EngramAccessService } from "./access-service.js";
|
|
5
|
+
import { OFFLINE_SYNC_MAX_MTIME_MS } from "./offline-sync.js";
|
|
5
6
|
|
|
6
7
|
function createOfflineService(): EngramAccessService {
|
|
7
8
|
return new EngramAccessService({
|
|
@@ -34,4 +35,20 @@ test("offline apply-file-content reports invalid metadata as input errors", asyn
|
|
|
34
35
|
error instanceof EngramAccessInputError &&
|
|
35
36
|
/sha256 must be a 64-character sha256/.test(error.message),
|
|
36
37
|
);
|
|
38
|
+
|
|
39
|
+
await assert.rejects(
|
|
40
|
+
() => service.offlineSyncApplyFileContent({
|
|
41
|
+
includeTranscripts: true,
|
|
42
|
+
sourceId: "laptop",
|
|
43
|
+
path: "state/lcm.sqlite",
|
|
44
|
+
sha256: "a".repeat(64),
|
|
45
|
+
bytes: 0,
|
|
46
|
+
mtimeMs: OFFLINE_SYNC_MAX_MTIME_MS + 1,
|
|
47
|
+
offset: 0,
|
|
48
|
+
content: Buffer.alloc(0),
|
|
49
|
+
}),
|
|
50
|
+
(error) =>
|
|
51
|
+
error instanceof EngramAccessInputError &&
|
|
52
|
+
/mtimeMs must be within JavaScript Date range/.test(error.message),
|
|
53
|
+
);
|
|
37
54
|
});
|
package/src/access-service.ts
CHANGED
|
@@ -139,11 +139,16 @@ import {
|
|
|
139
139
|
applyOfflineSyncFileContentChunk,
|
|
140
140
|
applyOfflineSyncChangeset,
|
|
141
141
|
buildOfflineSyncSnapshot,
|
|
142
|
+
buildOfflineSyncSnapshotFromBase,
|
|
142
143
|
buildOfflineSyncSnapshotForPaths,
|
|
144
|
+
iterateOfflineSyncSnapshotFileRecords,
|
|
145
|
+
OFFLINE_SYNC_SNAPSHOT_FORMAT,
|
|
143
146
|
readOfflineSyncFileContentChunk,
|
|
144
147
|
type OfflineSyncApplyFileContentChunkResult,
|
|
145
148
|
type OfflineSyncApplyChangesetResult,
|
|
149
|
+
type OfflineSyncFileRecord,
|
|
146
150
|
type OfflineSyncFileContentChunk,
|
|
151
|
+
type OfflineSyncFileState,
|
|
147
152
|
type OfflineSyncSnapshot,
|
|
148
153
|
} from "./offline-sync.js";
|
|
149
154
|
import {
|
|
@@ -619,6 +624,8 @@ export interface EngramAccessOfflineSyncSnapshotRequest {
|
|
|
619
624
|
principal?: string;
|
|
620
625
|
includeTranscripts?: boolean;
|
|
621
626
|
includeContent?: boolean;
|
|
627
|
+
baseCapturedAt?: Date;
|
|
628
|
+
baseFiles?: OfflineSyncFileState[];
|
|
622
629
|
}
|
|
623
630
|
|
|
624
631
|
export interface EngramAccessOfflineSyncFilesRequest {
|
|
@@ -655,12 +662,18 @@ export interface EngramAccessOfflineSyncApplyRequest {
|
|
|
655
662
|
namespace?: string;
|
|
656
663
|
principal?: string;
|
|
657
664
|
changeset: unknown;
|
|
665
|
+
returnCurrentFiles?: boolean;
|
|
658
666
|
}
|
|
659
667
|
|
|
660
668
|
export interface EngramAccessOfflineSyncSnapshotResponse extends OfflineSyncSnapshot {
|
|
661
669
|
namespace: string;
|
|
662
670
|
}
|
|
663
671
|
|
|
672
|
+
export interface EngramAccessOfflineSyncSnapshotStreamResponse extends Omit<OfflineSyncSnapshot, "files"> {
|
|
673
|
+
namespace: string;
|
|
674
|
+
files: AsyncIterable<OfflineSyncFileRecord>;
|
|
675
|
+
}
|
|
676
|
+
|
|
664
677
|
export interface EngramAccessOfflineSyncFilesResponse extends OfflineSyncSnapshot {
|
|
665
678
|
namespace: string;
|
|
666
679
|
}
|
|
@@ -5602,12 +5615,19 @@ export class EngramAccessService {
|
|
|
5602
5615
|
const resolvedNamespace = this.resolveReadableNamespace(options.namespace, options.principal);
|
|
5603
5616
|
const storage = await this.orchestrator.getStorage(resolvedNamespace);
|
|
5604
5617
|
const storageHash = createHash("sha256").update(storage.dir).digest("hex").slice(0, 16);
|
|
5605
|
-
const
|
|
5618
|
+
const snapshotBuilder = options.includeContent === false && options.baseFiles && options.baseFiles.length > 0
|
|
5619
|
+
? buildOfflineSyncSnapshotFromBase
|
|
5620
|
+
: buildOfflineSyncSnapshot;
|
|
5621
|
+
const snapshot = await snapshotBuilder({
|
|
5606
5622
|
root: storage.dir,
|
|
5607
5623
|
sourceId: `remnic:${resolvedNamespace}:${storageHash}`,
|
|
5624
|
+
...(options.baseFiles && options.baseFiles.length > 0 ? { baseFiles: options.baseFiles } : {}),
|
|
5625
|
+
// Client clocks are not authoritative for server-side ctime reuse. A
|
|
5626
|
+
// future client timestamp can hide same-size, preserved-mtime rewrites.
|
|
5608
5627
|
includeContent: options.includeContent !== false,
|
|
5609
5628
|
includeTranscripts: options.includeTranscripts !== false,
|
|
5610
5629
|
readFile: async ({ filePath }) => storage.readOfflineSyncFile(filePath),
|
|
5630
|
+
readFileDigest: async ({ filePath }) => storage.digestOfflineSyncFile(filePath),
|
|
5611
5631
|
});
|
|
5612
5632
|
return {
|
|
5613
5633
|
namespace: resolvedNamespace,
|
|
@@ -5615,6 +5635,29 @@ export class EngramAccessService {
|
|
|
5615
5635
|
};
|
|
5616
5636
|
}
|
|
5617
5637
|
|
|
5638
|
+
async offlineSyncSnapshotStream(
|
|
5639
|
+
options: Omit<EngramAccessOfflineSyncSnapshotRequest, "baseCapturedAt" | "baseFiles"> = {},
|
|
5640
|
+
): Promise<EngramAccessOfflineSyncSnapshotStreamResponse> {
|
|
5641
|
+
const resolvedNamespace = this.resolveReadableNamespace(options.namespace, options.principal);
|
|
5642
|
+
const storage = await this.orchestrator.getStorage(resolvedNamespace);
|
|
5643
|
+
const storageHash = createHash("sha256").update(storage.dir).digest("hex").slice(0, 16);
|
|
5644
|
+
return {
|
|
5645
|
+
namespace: resolvedNamespace,
|
|
5646
|
+
format: OFFLINE_SYNC_SNAPSHOT_FORMAT,
|
|
5647
|
+
schemaVersion: 1,
|
|
5648
|
+
createdAt: new Date().toISOString(),
|
|
5649
|
+
sourceId: `remnic:${resolvedNamespace}:${storageHash}`,
|
|
5650
|
+
includeTranscripts: options.includeTranscripts !== false,
|
|
5651
|
+
files: iterateOfflineSyncSnapshotFileRecords({
|
|
5652
|
+
root: storage.dir,
|
|
5653
|
+
includeContent: options.includeContent === true,
|
|
5654
|
+
includeTranscripts: options.includeTranscripts !== false,
|
|
5655
|
+
readFile: async ({ filePath }) => storage.readOfflineSyncFile(filePath),
|
|
5656
|
+
readFileDigest: async ({ filePath }) => storage.digestOfflineSyncFile(filePath),
|
|
5657
|
+
}),
|
|
5658
|
+
};
|
|
5659
|
+
}
|
|
5660
|
+
|
|
5618
5661
|
async offlineSyncFiles(
|
|
5619
5662
|
options: EngramAccessOfflineSyncFilesRequest,
|
|
5620
5663
|
): Promise<EngramAccessOfflineSyncFilesResponse> {
|
|
@@ -5702,6 +5745,7 @@ export class EngramAccessService {
|
|
|
5702
5745
|
content: options.content,
|
|
5703
5746
|
includeTranscripts: options.includeTranscripts !== false,
|
|
5704
5747
|
readFile: async ({ filePath }) => storage.readOfflineSyncFile(filePath),
|
|
5748
|
+
readFileDigest: async ({ filePath }) => storage.digestOfflineSyncFile(filePath),
|
|
5705
5749
|
writeFile: async ({ filePath, content }) => storage.writeOfflineSyncFile(filePath, content),
|
|
5706
5750
|
writeStagingFile: async ({ filePath, content }) => storage.writeOfflineSyncStagingFile(filePath, content),
|
|
5707
5751
|
writeFileChunks: async ({ filePath, chunks }) => storage.writeOfflineSyncFileChunks(filePath, chunks),
|
|
@@ -5743,7 +5787,9 @@ export class EngramAccessService {
|
|
|
5743
5787
|
const result = await applyOfflineSyncChangeset({
|
|
5744
5788
|
root: storage.dir,
|
|
5745
5789
|
changeset: options.changeset,
|
|
5790
|
+
returnCurrentFiles: options.returnCurrentFiles,
|
|
5746
5791
|
readFile: async ({ filePath }) => storage.readOfflineSyncFile(filePath),
|
|
5792
|
+
readFileDigest: async ({ filePath }) => storage.digestOfflineSyncFile(filePath),
|
|
5747
5793
|
writeFile: async ({ filePath, content }) => storage.writeOfflineSyncFile(filePath, content),
|
|
5748
5794
|
deleteFile: async ({ filePath }) => storage.deleteOfflineSyncFile(filePath),
|
|
5749
5795
|
});
|
package/src/index.ts
CHANGED
|
@@ -683,23 +683,29 @@ export {
|
|
|
683
683
|
OFFLINE_SYNC_CHANGESET_FORMAT,
|
|
684
684
|
OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES,
|
|
685
685
|
OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES,
|
|
686
|
+
OFFLINE_SYNC_SNAPSHOT_BASE_MAX_BODY_BYTES,
|
|
686
687
|
OFFLINE_SYNC_SNAPSHOT_FORMAT,
|
|
687
688
|
OFFLINE_SYNC_STATE_VERSION,
|
|
688
689
|
applyOfflineSyncFileContentChunk,
|
|
689
690
|
applyOfflineSyncChangeset,
|
|
690
691
|
applyOfflineSyncSnapshot,
|
|
691
692
|
buildOfflineSyncChangeset,
|
|
693
|
+
buildOfflineSyncChangesetFromSnapshot,
|
|
692
694
|
buildOfflineSyncSnapshot,
|
|
695
|
+
buildOfflineSyncSnapshotFromBase,
|
|
693
696
|
buildOfflineSyncSnapshotForPaths,
|
|
694
697
|
defaultOfflineSyncStatePath,
|
|
695
698
|
fileStatesFromSnapshot,
|
|
699
|
+
iterateOfflineSyncSnapshotFileRecords,
|
|
696
700
|
normalizeOfflineSyncChangeset,
|
|
697
701
|
normalizeOfflineSyncSnapshot,
|
|
698
702
|
offlineSyncStateFromSnapshot,
|
|
699
703
|
readOfflineSyncFileContentChunk,
|
|
700
704
|
readOfflineSyncState,
|
|
705
|
+
shouldPreferIncomingOfflineRuntimeFile,
|
|
701
706
|
summarizeOfflineSyncChangeset,
|
|
702
707
|
summarizeOfflineSyncPendingChanges,
|
|
708
|
+
summarizeOfflineSyncPendingFiles,
|
|
703
709
|
writeOfflineSyncState,
|
|
704
710
|
type OfflineSyncApplyFileContentChunkResult,
|
|
705
711
|
type OfflineSyncApplyChangesetResult,
|
|
@@ -707,6 +713,7 @@ export {
|
|
|
707
713
|
type OfflineSyncChange,
|
|
708
714
|
type OfflineSyncChangeset,
|
|
709
715
|
type OfflineSyncConflict,
|
|
716
|
+
type OfflineSyncFileDigest,
|
|
710
717
|
type OfflineSyncFileRecord,
|
|
711
718
|
type OfflineSyncFileContentChunk,
|
|
712
719
|
type OfflineSyncFileState,
|