@remnic/cli 1.0.16 → 1.0.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +458 -9
- package/package.json +6 -6
package/dist/index.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
import fs7 from "fs";
|
|
3
3
|
import os from "os";
|
|
4
4
|
import path11 from "path";
|
|
5
|
-
import { createHash } from "crypto";
|
|
5
|
+
import { createDecipheriv, createHash } from "crypto";
|
|
6
6
|
import * as childProcess2 from "child_process";
|
|
7
7
|
import { fileURLToPath as fileURLToPath4 } from "url";
|
|
8
8
|
import {
|
|
@@ -88,15 +88,17 @@ import {
|
|
|
88
88
|
formatProcedureStatsText,
|
|
89
89
|
parseXrayCliOptions,
|
|
90
90
|
renderXray,
|
|
91
|
+
OFFLINE_SYNC_APPLY_MAX_BODY_BYTES,
|
|
91
92
|
OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES,
|
|
93
|
+
OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES,
|
|
92
94
|
applyOfflineSyncSnapshot,
|
|
93
95
|
buildOfflineSyncChangeset,
|
|
94
96
|
buildOfflineSyncSnapshot,
|
|
95
97
|
defaultOfflineSyncStatePath,
|
|
96
98
|
normalizeOfflineSyncSnapshot,
|
|
97
99
|
offlineSyncStateFromSnapshot,
|
|
100
|
+
readOfflineSyncFileContentChunk,
|
|
98
101
|
readOfflineSyncState,
|
|
99
|
-
summarizeOfflineSyncChangeset,
|
|
100
102
|
summarizeOfflineSyncPendingChanges,
|
|
101
103
|
writeOfflineSyncState,
|
|
102
104
|
buildActionConfidenceInputFromOptions,
|
|
@@ -106,7 +108,24 @@ import {
|
|
|
106
108
|
forkCapsule,
|
|
107
109
|
readForkLineage
|
|
108
110
|
} from "@remnic/core";
|
|
109
|
-
import {
|
|
111
|
+
import {
|
|
112
|
+
AUTH_TAG_LENGTH,
|
|
113
|
+
ENVELOPE_HEADER_SIZE,
|
|
114
|
+
ENVELOPE_LAYOUT,
|
|
115
|
+
ENVELOPE_SALT_LENGTH,
|
|
116
|
+
ENVELOPE_VERSION,
|
|
117
|
+
FILE_FORMAT_FLAGS,
|
|
118
|
+
FILE_FORMAT_VERSION,
|
|
119
|
+
IV_LENGTH,
|
|
120
|
+
MAGIC_BYTES,
|
|
121
|
+
MAGIC_HEADER_SIZE,
|
|
122
|
+
SecureStoreLockedError,
|
|
123
|
+
filePathAad,
|
|
124
|
+
isEncryptedFile,
|
|
125
|
+
keyring,
|
|
126
|
+
readHeader,
|
|
127
|
+
secureStoreDir
|
|
128
|
+
} from "@remnic/core/secure-store";
|
|
110
129
|
|
|
111
130
|
// src/optional-module-loader.ts
|
|
112
131
|
function isSpecifierNotFoundError(err, specifier) {
|
|
@@ -6672,6 +6691,16 @@ async function fetchOfflineFiles(args) {
|
|
|
6672
6691
|
}
|
|
6673
6692
|
var OFFLINE_SYNC_DIRECT_HYDRATE_MIN_BYTES = 16 * 1024 * 1024;
|
|
6674
6693
|
var OFFLINE_SYNC_FILES_CONTENT_MAX_BATCH_BYTES = 8 * 1024 * 1024;
|
|
6694
|
+
var OFFLINE_SYNC_APPLY_MAX_REQUEST_BYTES = Math.floor(OFFLINE_SYNC_APPLY_MAX_BODY_BYTES / 2);
|
|
6695
|
+
var OFFLINE_SYNC_DIRECT_PUSH_INLINE_MARGIN_BYTES = 256 * 1024;
|
|
6696
|
+
var OFFLINE_SYNC_INLINE_CONTENT_MAX_BYTES = Math.max(
|
|
6697
|
+
1,
|
|
6698
|
+
Math.floor((OFFLINE_SYNC_APPLY_MAX_REQUEST_BYTES - OFFLINE_SYNC_DIRECT_PUSH_INLINE_MARGIN_BYTES) * 3 / 4)
|
|
6699
|
+
);
|
|
6700
|
+
var OFFLINE_SYNC_DIRECT_PUSH_MIN_BYTES = Math.min(
|
|
6701
|
+
OFFLINE_SYNC_DIRECT_HYDRATE_MIN_BYTES,
|
|
6702
|
+
OFFLINE_SYNC_INLINE_CONTENT_MAX_BYTES
|
|
6703
|
+
);
|
|
6675
6704
|
function parseOfflineHeaderNumber(headers, name) {
|
|
6676
6705
|
const raw = headers.get(name);
|
|
6677
6706
|
if (raw === null) throw new Error(`offline file content response omitted ${name}`);
|
|
@@ -6728,6 +6757,41 @@ async function fetchOfflineFileContentChunk(args) {
|
|
|
6728
6757
|
content
|
|
6729
6758
|
};
|
|
6730
6759
|
}
|
|
6760
|
+
async function postOfflineFileContentChunk(args) {
|
|
6761
|
+
const response = await fetch(
|
|
6762
|
+
offlineEndpoint(args.remoteUrl, "/remnic/v1/offline-sync/apply-file-content", {
|
|
6763
|
+
namespace: args.namespace
|
|
6764
|
+
}),
|
|
6765
|
+
{
|
|
6766
|
+
method: "POST",
|
|
6767
|
+
headers: {
|
|
6768
|
+
authorization: `Bearer ${args.token}`,
|
|
6769
|
+
"content-type": "application/octet-stream",
|
|
6770
|
+
"x-remnic-include-transcripts": args.includeTranscripts ? "true" : "false",
|
|
6771
|
+
"x-remnic-source-id": encodeURIComponent(args.sourceId),
|
|
6772
|
+
"x-remnic-file-path": encodeURIComponent(args.file.path),
|
|
6773
|
+
"x-remnic-file-sha256": args.file.sha256,
|
|
6774
|
+
"x-remnic-file-bytes": String(args.file.bytes),
|
|
6775
|
+
"x-remnic-file-mtime-ms": String(args.file.mtimeMs),
|
|
6776
|
+
"x-remnic-chunk-offset": String(args.offset),
|
|
6777
|
+
...args.baseSha256 ? { "x-remnic-base-sha256": args.baseSha256 } : {}
|
|
6778
|
+
},
|
|
6779
|
+
body: new Blob([new Uint8Array(args.content)])
|
|
6780
|
+
}
|
|
6781
|
+
);
|
|
6782
|
+
if (!response.ok) {
|
|
6783
|
+
let detail = "";
|
|
6784
|
+
try {
|
|
6785
|
+
detail = await response.text();
|
|
6786
|
+
} catch {
|
|
6787
|
+
detail = "";
|
|
6788
|
+
}
|
|
6789
|
+
throw new Error(
|
|
6790
|
+
`offline sync apply-file-content request failed: ${response.status} ${response.statusText}${detail ? ` - ${detail.slice(0, 500)}` : ""}`
|
|
6791
|
+
);
|
|
6792
|
+
}
|
|
6793
|
+
return await response.json();
|
|
6794
|
+
}
|
|
6731
6795
|
function resolvedOfflineSnapshotNamespace(snapshot, requestedNamespace) {
|
|
6732
6796
|
const resolved = typeof snapshot.namespace === "string" && snapshot.namespace.trim().length > 0 ? snapshot.namespace.trim() : void 0;
|
|
6733
6797
|
return resolved ?? requestedNamespace;
|
|
@@ -6782,6 +6846,13 @@ function shouldDirectHydrateOfflineFile(options) {
|
|
|
6782
6846
|
}
|
|
6783
6847
|
return !options.current && !options.base;
|
|
6784
6848
|
}
|
|
6849
|
+
function offlineDirectPushFiles(options) {
|
|
6850
|
+
const base = offlineFileStateMap(options.baseFiles);
|
|
6851
|
+
return options.currentFiles.filter((current) => {
|
|
6852
|
+
if (current.bytes < OFFLINE_SYNC_DIRECT_PUSH_MIN_BYTES) return false;
|
|
6853
|
+
return current.sha256 !== base.get(current.path)?.sha256;
|
|
6854
|
+
}).sort((left, right) => right.bytes - left.bytes || left.path.localeCompare(right.path));
|
|
6855
|
+
}
|
|
6785
6856
|
function resolveOfflineDirectHydrationPath(memoryDir, relPath) {
|
|
6786
6857
|
const base = path11.resolve(memoryDir);
|
|
6787
6858
|
const target = path11.resolve(base, relPath);
|
|
@@ -6791,6 +6862,134 @@ function resolveOfflineDirectHydrationPath(memoryDir, relPath) {
|
|
|
6791
6862
|
}
|
|
6792
6863
|
return target;
|
|
6793
6864
|
}
|
|
6865
|
+
var OFFLINE_SYNC_FILE_CONTENT_UPLOAD_CHUNK_BYTES = OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES;
|
|
6866
|
+
async function pushOfflineFileContent(args) {
|
|
6867
|
+
if (args.readFileChunks) {
|
|
6868
|
+
return pushOfflineFileContentFromChunkReader(args);
|
|
6869
|
+
}
|
|
6870
|
+
let offset = 0;
|
|
6871
|
+
let finalResult = null;
|
|
6872
|
+
while (offset < args.file.bytes || args.file.bytes === 0 && offset === 0) {
|
|
6873
|
+
const chunk = await readOfflineSyncFileContentChunk({
|
|
6874
|
+
root: args.memoryDir,
|
|
6875
|
+
path: args.file.path,
|
|
6876
|
+
offset,
|
|
6877
|
+
length: Math.min(
|
|
6878
|
+
OFFLINE_SYNC_FILE_CONTENT_UPLOAD_CHUNK_BYTES,
|
|
6879
|
+
Math.max(1, args.file.bytes - offset)
|
|
6880
|
+
),
|
|
6881
|
+
includeTranscripts: args.includeTranscripts,
|
|
6882
|
+
readFile: args.readFile
|
|
6883
|
+
});
|
|
6884
|
+
if (chunk.path !== args.file.path || chunk.sha256 !== void 0 && chunk.sha256 !== args.file.sha256 || chunk.bytes !== args.file.bytes || chunk.mtimeMs !== args.file.mtimeMs || chunk.offset !== offset) {
|
|
6885
|
+
throw new Error(`local file changed while pushing offline content: ${args.file.path}`);
|
|
6886
|
+
}
|
|
6887
|
+
if (chunk.chunkBytes === 0 && args.file.bytes > 0) {
|
|
6888
|
+
throw new Error(`local offline content chunk was empty before EOF: ${args.file.path}`);
|
|
6889
|
+
}
|
|
6890
|
+
finalResult = await postOfflineFileContentChunk({
|
|
6891
|
+
remoteUrl: args.remoteUrl,
|
|
6892
|
+
token: args.token,
|
|
6893
|
+
namespace: args.namespace,
|
|
6894
|
+
includeTranscripts: args.includeTranscripts,
|
|
6895
|
+
sourceId: args.sourceId,
|
|
6896
|
+
file: args.file,
|
|
6897
|
+
baseSha256: args.baseSha256,
|
|
6898
|
+
offset,
|
|
6899
|
+
content: chunk.content
|
|
6900
|
+
});
|
|
6901
|
+
if (finalResult.conflict) {
|
|
6902
|
+
return finalResult;
|
|
6903
|
+
}
|
|
6904
|
+
offset += chunk.chunkBytes;
|
|
6905
|
+
if (args.file.bytes === 0) break;
|
|
6906
|
+
}
|
|
6907
|
+
if (!finalResult?.done) {
|
|
6908
|
+
throw new Error(`offline sync large-file push did not finish for ${args.file.path}`);
|
|
6909
|
+
}
|
|
6910
|
+
return finalResult;
|
|
6911
|
+
}
|
|
6912
|
+
async function pushOfflineFileContentFromChunkReader(args) {
|
|
6913
|
+
const filePath = resolveOfflineDirectHydrationPath(args.memoryDir, args.file.path);
|
|
6914
|
+
const stat = fs7.statSync(filePath);
|
|
6915
|
+
if (stat.mtimeMs !== args.file.mtimeMs) {
|
|
6916
|
+
throw new Error(`local file changed while pushing offline content: ${args.file.path}`);
|
|
6917
|
+
}
|
|
6918
|
+
const hash = createHash("sha256");
|
|
6919
|
+
const chunks = args.readFileChunks({
|
|
6920
|
+
root: path11.resolve(args.memoryDir),
|
|
6921
|
+
path: args.file.path,
|
|
6922
|
+
filePath,
|
|
6923
|
+
chunkSize: OFFLINE_SYNC_FILE_CONTENT_UPLOAD_CHUNK_BYTES
|
|
6924
|
+
});
|
|
6925
|
+
let offset = 0;
|
|
6926
|
+
let pending = null;
|
|
6927
|
+
let finalResult = null;
|
|
6928
|
+
for await (const rawChunk of chunks) {
|
|
6929
|
+
const chunk = Buffer.from(rawChunk);
|
|
6930
|
+
if (chunk.length === 0) continue;
|
|
6931
|
+
if (chunk.length > OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES) {
|
|
6932
|
+
throw new Error(`local offline content chunk exceeds max size: ${args.file.path}`);
|
|
6933
|
+
}
|
|
6934
|
+
if (pending) {
|
|
6935
|
+
hash.update(pending);
|
|
6936
|
+
finalResult = await postOfflineFileContentChunk({
|
|
6937
|
+
remoteUrl: args.remoteUrl,
|
|
6938
|
+
token: args.token,
|
|
6939
|
+
namespace: args.namespace,
|
|
6940
|
+
includeTranscripts: args.includeTranscripts,
|
|
6941
|
+
sourceId: args.sourceId,
|
|
6942
|
+
file: args.file,
|
|
6943
|
+
baseSha256: args.baseSha256,
|
|
6944
|
+
offset,
|
|
6945
|
+
content: pending
|
|
6946
|
+
});
|
|
6947
|
+
if (finalResult.conflict) {
|
|
6948
|
+
return finalResult;
|
|
6949
|
+
}
|
|
6950
|
+
offset += pending.length;
|
|
6951
|
+
}
|
|
6952
|
+
pending = chunk;
|
|
6953
|
+
}
|
|
6954
|
+
if (pending) hash.update(pending);
|
|
6955
|
+
const finalBytes = offset + (pending?.length ?? 0);
|
|
6956
|
+
const digest = hash.digest("hex");
|
|
6957
|
+
if (digest !== args.file.sha256 || finalBytes !== args.file.bytes) {
|
|
6958
|
+
throw new Error(`local file changed while pushing offline content: ${args.file.path}`);
|
|
6959
|
+
}
|
|
6960
|
+
if (pending) {
|
|
6961
|
+
finalResult = await postOfflineFileContentChunk({
|
|
6962
|
+
remoteUrl: args.remoteUrl,
|
|
6963
|
+
token: args.token,
|
|
6964
|
+
namespace: args.namespace,
|
|
6965
|
+
includeTranscripts: args.includeTranscripts,
|
|
6966
|
+
sourceId: args.sourceId,
|
|
6967
|
+
file: args.file,
|
|
6968
|
+
baseSha256: args.baseSha256,
|
|
6969
|
+
offset,
|
|
6970
|
+
content: pending
|
|
6971
|
+
});
|
|
6972
|
+
if (finalResult.conflict) {
|
|
6973
|
+
return finalResult;
|
|
6974
|
+
}
|
|
6975
|
+
} else if (args.file.bytes === 0) {
|
|
6976
|
+
finalResult = await postOfflineFileContentChunk({
|
|
6977
|
+
remoteUrl: args.remoteUrl,
|
|
6978
|
+
token: args.token,
|
|
6979
|
+
namespace: args.namespace,
|
|
6980
|
+
includeTranscripts: args.includeTranscripts,
|
|
6981
|
+
sourceId: args.sourceId,
|
|
6982
|
+
file: args.file,
|
|
6983
|
+
baseSha256: args.baseSha256,
|
|
6984
|
+
offset: 0,
|
|
6985
|
+
content: Buffer.alloc(0)
|
|
6986
|
+
});
|
|
6987
|
+
}
|
|
6988
|
+
if (!finalResult?.done) {
|
|
6989
|
+
throw new Error(`offline sync large-file push did not finish for ${args.file.path}`);
|
|
6990
|
+
}
|
|
6991
|
+
return finalResult;
|
|
6992
|
+
}
|
|
6794
6993
|
async function fetchOfflineFileContent(args) {
|
|
6795
6994
|
const chunks = [];
|
|
6796
6995
|
const hash = createHash("sha256");
|
|
@@ -6940,7 +7139,39 @@ async function hydrateOfflineSnapshotContent(args) {
|
|
|
6940
7139
|
})
|
|
6941
7140
|
};
|
|
6942
7141
|
}
|
|
6943
|
-
|
|
7142
|
+
function chunkOfflineChangesetApplyBatches(changeset, namespace, maxRequestBytes = OFFLINE_SYNC_APPLY_MAX_REQUEST_BYTES) {
|
|
7143
|
+
if (!Number.isInteger(maxRequestBytes) || maxRequestBytes < 1) {
|
|
7144
|
+
throw new Error("offline sync apply max request bytes must be a positive integer");
|
|
7145
|
+
}
|
|
7146
|
+
const chunks = [];
|
|
7147
|
+
let current = [];
|
|
7148
|
+
const requestBytesFor = (changes) => Buffer.byteLength(JSON.stringify({
|
|
7149
|
+
namespace,
|
|
7150
|
+
changeset: {
|
|
7151
|
+
...changeset,
|
|
7152
|
+
changes
|
|
7153
|
+
}
|
|
7154
|
+
}), "utf-8");
|
|
7155
|
+
for (const change of changeset.changes) {
|
|
7156
|
+
const withChange = [...current, change];
|
|
7157
|
+
if (current.length > 0 && requestBytesFor(withChange) > maxRequestBytes) {
|
|
7158
|
+
chunks.push({ ...changeset, changes: current });
|
|
7159
|
+
current = [];
|
|
7160
|
+
}
|
|
7161
|
+
const singleBytes = requestBytesFor([...current, change]);
|
|
7162
|
+
if (singleBytes > maxRequestBytes) {
|
|
7163
|
+
throw new Error(
|
|
7164
|
+
`offline sync change for ${change.path} exceeds the apply request size budget; retry after direct-push threshold is lowered`
|
|
7165
|
+
);
|
|
7166
|
+
}
|
|
7167
|
+
current.push(change);
|
|
7168
|
+
}
|
|
7169
|
+
if (current.length > 0) {
|
|
7170
|
+
chunks.push({ ...changeset, changes: current });
|
|
7171
|
+
}
|
|
7172
|
+
return chunks;
|
|
7173
|
+
}
|
|
7174
|
+
async function postOfflineChangesBatch(args) {
|
|
6944
7175
|
return fetchOfflineJson(
|
|
6945
7176
|
offlineEndpoint(args.remoteUrl, "/remnic/v1/offline-sync/apply"),
|
|
6946
7177
|
args.token,
|
|
@@ -6953,6 +7184,31 @@ async function pushOfflineChanges(args) {
|
|
|
6953
7184
|
}
|
|
6954
7185
|
);
|
|
6955
7186
|
}
|
|
7187
|
+
async function pushOfflineChanges(args) {
|
|
7188
|
+
let namespace = args.namespace ?? "";
|
|
7189
|
+
let appliedUpserts = 0;
|
|
7190
|
+
let appliedDeletes = 0;
|
|
7191
|
+
let skipped = 0;
|
|
7192
|
+
const conflicts = [];
|
|
7193
|
+
for (const changeset of chunkOfflineChangesetApplyBatches(args.changeset, args.namespace)) {
|
|
7194
|
+
const result = await postOfflineChangesBatch({
|
|
7195
|
+
...args,
|
|
7196
|
+
changeset
|
|
7197
|
+
});
|
|
7198
|
+
namespace = result.namespace || namespace;
|
|
7199
|
+
appliedUpserts += result.appliedUpserts;
|
|
7200
|
+
appliedDeletes += result.appliedDeletes;
|
|
7201
|
+
skipped += result.skipped;
|
|
7202
|
+
conflicts.push(...result.conflicts);
|
|
7203
|
+
}
|
|
7204
|
+
return {
|
|
7205
|
+
namespace,
|
|
7206
|
+
appliedUpserts,
|
|
7207
|
+
appliedDeletes,
|
|
7208
|
+
skipped,
|
|
7209
|
+
conflicts
|
|
7210
|
+
};
|
|
7211
|
+
}
|
|
6956
7212
|
function parseOfflineIntervalMs(args) {
|
|
6957
7213
|
const raw = resolveRequiredValueFlag(args, "--interval-ms");
|
|
6958
7214
|
if (raw === void 0) return 6e4;
|
|
@@ -6978,19 +7234,137 @@ function waitForOfflineInterval(ms, setCancel) {
|
|
|
6978
7234
|
async function createOfflineStorageIo(memoryDir) {
|
|
6979
7235
|
const storage = new StorageManager(memoryDir);
|
|
6980
7236
|
const header = await readHeader(memoryDir);
|
|
7237
|
+
let secureStoreKey = null;
|
|
6981
7238
|
if (header) {
|
|
6982
7239
|
storage.setSecureStoreRequired(true);
|
|
6983
7240
|
const key = keyring.getKey(secureStoreDir(memoryDir));
|
|
6984
7241
|
if (key) {
|
|
6985
7242
|
storage.setSecureStoreKey(key);
|
|
7243
|
+
secureStoreKey = key;
|
|
6986
7244
|
}
|
|
6987
7245
|
}
|
|
6988
7246
|
return {
|
|
6989
7247
|
readFile: async ({ filePath }) => storage.readOfflineSyncFile(filePath),
|
|
7248
|
+
readFileChunks: ({ filePath, chunkSize }) => readOfflineSyncFileChunks({
|
|
7249
|
+
filePath,
|
|
7250
|
+
memoryDir,
|
|
7251
|
+
secureStoreKey,
|
|
7252
|
+
chunkSize
|
|
7253
|
+
}),
|
|
6990
7254
|
writeFile: async ({ filePath, content }) => storage.writeOfflineSyncFile(filePath, content),
|
|
6991
7255
|
deleteFile: async ({ filePath }) => storage.deleteOfflineSyncFile(filePath)
|
|
6992
7256
|
};
|
|
6993
7257
|
}
|
|
7258
|
+
async function* readOfflineSyncFileChunks(options) {
|
|
7259
|
+
const header = await readFilePrefix(options.filePath, MAGIC_HEADER_SIZE);
|
|
7260
|
+
if (!isEncryptedFile(header)) {
|
|
7261
|
+
yield* readPlainOfflineFileChunks(options.filePath, options.chunkSize);
|
|
7262
|
+
return;
|
|
7263
|
+
}
|
|
7264
|
+
if (!options.secureStoreKey) {
|
|
7265
|
+
throw new SecureStoreLockedError(
|
|
7266
|
+
`secure-store is locked \u2014 cannot read encrypted file at ${options.filePath}. Run \`remnic secure-store unlock\` to decrypt.`
|
|
7267
|
+
);
|
|
7268
|
+
}
|
|
7269
|
+
yield* readEncryptedOfflineFileChunks({
|
|
7270
|
+
filePath: options.filePath,
|
|
7271
|
+
memoryDir: options.memoryDir,
|
|
7272
|
+
key: options.secureStoreKey,
|
|
7273
|
+
chunkSize: options.chunkSize
|
|
7274
|
+
});
|
|
7275
|
+
}
|
|
7276
|
+
async function readFilePrefix(filePath, length) {
|
|
7277
|
+
const handle = await fs7.promises.open(filePath, "r");
|
|
7278
|
+
try {
|
|
7279
|
+
const out = Buffer.alloc(length);
|
|
7280
|
+
const { bytesRead } = await handle.read(out, 0, length, 0);
|
|
7281
|
+
return out.subarray(0, bytesRead);
|
|
7282
|
+
} finally {
|
|
7283
|
+
await handle.close();
|
|
7284
|
+
}
|
|
7285
|
+
}
|
|
7286
|
+
async function* readPlainOfflineFileChunks(filePath, chunkSize) {
|
|
7287
|
+
const stream = fs7.createReadStream(filePath, { highWaterMark: chunkSize });
|
|
7288
|
+
for await (const chunk of stream) {
|
|
7289
|
+
yield Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
|
7290
|
+
}
|
|
7291
|
+
}
|
|
7292
|
+
async function* readEncryptedOfflineFileChunks(options) {
|
|
7293
|
+
const header = await readFilePrefix(options.filePath, MAGIC_HEADER_SIZE + ENVELOPE_HEADER_SIZE);
|
|
7294
|
+
if (header.length < MAGIC_HEADER_SIZE + ENVELOPE_HEADER_SIZE || !isEncryptedFile(header)) {
|
|
7295
|
+
throw new Error(`secure-store encrypted file is truncated: ${options.filePath}`);
|
|
7296
|
+
}
|
|
7297
|
+
const version = header.readUInt8(MAGIC_BYTES.length);
|
|
7298
|
+
const flags = header.readUInt8(MAGIC_BYTES.length + 1);
|
|
7299
|
+
if (version !== FILE_FORMAT_VERSION) {
|
|
7300
|
+
throw new Error(`secure-store file has unsupported version ${version}: ${options.filePath}`);
|
|
7301
|
+
}
|
|
7302
|
+
if (flags !== FILE_FORMAT_FLAGS) {
|
|
7303
|
+
throw new Error(`secure-store file has unsupported flags 0x${flags.toString(16)}: ${options.filePath}`);
|
|
7304
|
+
}
|
|
7305
|
+
const envelopeHeader = header.subarray(MAGIC_HEADER_SIZE);
|
|
7306
|
+
const envelopeVersion = envelopeHeader.readUInt8(ENVELOPE_LAYOUT.version);
|
|
7307
|
+
if (envelopeVersion !== ENVELOPE_VERSION) {
|
|
7308
|
+
throw new Error(`secure-store envelope has unsupported version ${envelopeVersion}: ${options.filePath}`);
|
|
7309
|
+
}
|
|
7310
|
+
const salt = envelopeHeader.subarray(
|
|
7311
|
+
ENVELOPE_LAYOUT.salt,
|
|
7312
|
+
ENVELOPE_LAYOUT.salt + ENVELOPE_SALT_LENGTH
|
|
7313
|
+
);
|
|
7314
|
+
const iv = envelopeHeader.subarray(ENVELOPE_LAYOUT.iv, ENVELOPE_LAYOUT.iv + IV_LENGTH);
|
|
7315
|
+
const authTag = envelopeHeader.subarray(
|
|
7316
|
+
ENVELOPE_LAYOUT.authTag,
|
|
7317
|
+
ENVELOPE_LAYOUT.authTag + AUTH_TAG_LENGTH
|
|
7318
|
+
);
|
|
7319
|
+
const decipher = createDecipheriv("aes-256-gcm", options.key, iv, {
|
|
7320
|
+
authTagLength: AUTH_TAG_LENGTH
|
|
7321
|
+
});
|
|
7322
|
+
decipher.setAuthTag(authTag);
|
|
7323
|
+
decipher.setAAD(Buffer.concat([secureStoreEnvelopeHeaderAad(salt), filePathAad(options.filePath, options.memoryDir)]));
|
|
7324
|
+
let pending = Buffer.alloc(0);
|
|
7325
|
+
const stream = fs7.createReadStream(options.filePath, {
|
|
7326
|
+
start: MAGIC_HEADER_SIZE + ENVELOPE_HEADER_SIZE,
|
|
7327
|
+
highWaterMark: options.chunkSize
|
|
7328
|
+
});
|
|
7329
|
+
for await (const encryptedChunk of stream) {
|
|
7330
|
+
const plain = decipher.update(Buffer.isBuffer(encryptedChunk) ? encryptedChunk : Buffer.from(encryptedChunk));
|
|
7331
|
+
if (plain.length > 0) {
|
|
7332
|
+
pending = Buffer.concat([pending, plain], pending.length + plain.length);
|
|
7333
|
+
}
|
|
7334
|
+
while (pending.length >= options.chunkSize) {
|
|
7335
|
+
yield pending.subarray(0, options.chunkSize);
|
|
7336
|
+
pending = pending.subarray(options.chunkSize);
|
|
7337
|
+
}
|
|
7338
|
+
}
|
|
7339
|
+
const finalPlain = decipher.final();
|
|
7340
|
+
if (finalPlain.length > 0) {
|
|
7341
|
+
pending = Buffer.concat([pending, finalPlain], pending.length + finalPlain.length);
|
|
7342
|
+
}
|
|
7343
|
+
while (pending.length >= options.chunkSize) {
|
|
7344
|
+
yield pending.subarray(0, options.chunkSize);
|
|
7345
|
+
pending = pending.subarray(options.chunkSize);
|
|
7346
|
+
}
|
|
7347
|
+
if (pending.length > 0) yield pending;
|
|
7348
|
+
}
|
|
7349
|
+
function secureStoreEnvelopeHeaderAad(salt) {
|
|
7350
|
+
const out = Buffer.alloc(1 + ENVELOPE_SALT_LENGTH);
|
|
7351
|
+
out.writeUInt8(ENVELOPE_VERSION, 0);
|
|
7352
|
+
Buffer.from(salt).copy(out, 1);
|
|
7353
|
+
return out;
|
|
7354
|
+
}
|
|
7355
|
+
function formatOfflineLargeFilePushFailureMessage(failures) {
|
|
7356
|
+
const paths = failures.slice(0, 5).map((failure) => `${failure.path}: ${failure.error}`).join("; ");
|
|
7357
|
+
const suffix = failures.length > 5 ? `; +${failures.length - 5} more` : "";
|
|
7358
|
+
return `offline sync large-file push failed for ${failures.length} file${failures.length === 1 ? "" : "s"}: ${paths}${suffix}`;
|
|
7359
|
+
}
|
|
7360
|
+
var OfflineLargeFilePushError = class extends Error {
|
|
7361
|
+
failures;
|
|
7362
|
+
constructor(failures) {
|
|
7363
|
+
super(formatOfflineLargeFilePushFailureMessage(failures));
|
|
7364
|
+
this.name = "OfflineLargeFilePushError";
|
|
7365
|
+
this.failures = failures;
|
|
7366
|
+
}
|
|
7367
|
+
};
|
|
6994
7368
|
async function runOfflineSyncOnce(options) {
|
|
6995
7369
|
fs7.mkdirSync(options.memoryDir, { recursive: true });
|
|
6996
7370
|
let activeStatePath = options.statePath;
|
|
@@ -7030,20 +7404,90 @@ async function runOfflineSyncOnce(options) {
|
|
|
7030
7404
|
}
|
|
7031
7405
|
const baseFiles = priorState?.baseFiles ?? [];
|
|
7032
7406
|
const storageIo = await createOfflineStorageIo(options.memoryDir);
|
|
7407
|
+
const localSourceId = localOfflineSourceId(options.memoryDir);
|
|
7408
|
+
const pendingSummary = await summarizeOfflineSyncPendingChanges({
|
|
7409
|
+
root: options.memoryDir,
|
|
7410
|
+
sourceId: localSourceId,
|
|
7411
|
+
baseFiles,
|
|
7412
|
+
includeTranscripts: options.includeTranscripts,
|
|
7413
|
+
readFile: storageIo.readFile
|
|
7414
|
+
});
|
|
7415
|
+
const currentSnapshotForPush = await buildOfflineSyncSnapshot({
|
|
7416
|
+
root: options.memoryDir,
|
|
7417
|
+
sourceId: localSourceId,
|
|
7418
|
+
includeContent: false,
|
|
7419
|
+
includeTranscripts: options.includeTranscripts,
|
|
7420
|
+
readFile: storageIo.readFile
|
|
7421
|
+
});
|
|
7422
|
+
const baseByPath = offlineFileStateMap(baseFiles);
|
|
7423
|
+
let directPushAppliedUpserts = 0;
|
|
7424
|
+
let directPushSkipped = 0;
|
|
7425
|
+
let directPushNamespace;
|
|
7426
|
+
const directPushConflicts = [];
|
|
7427
|
+
const directPushedPaths = /* @__PURE__ */ new Set();
|
|
7428
|
+
const directPushFailures = [];
|
|
7429
|
+
for (const file of offlineDirectPushFiles({
|
|
7430
|
+
currentFiles: currentSnapshotForPush.files,
|
|
7431
|
+
baseFiles
|
|
7432
|
+
})) {
|
|
7433
|
+
let result;
|
|
7434
|
+
try {
|
|
7435
|
+
result = await pushOfflineFileContent({
|
|
7436
|
+
remoteUrl: options.remoteUrl,
|
|
7437
|
+
token: options.token,
|
|
7438
|
+
namespace: syncNamespace,
|
|
7439
|
+
includeTranscripts: options.includeTranscripts,
|
|
7440
|
+
memoryDir: options.memoryDir,
|
|
7441
|
+
sourceId: localSourceId,
|
|
7442
|
+
file,
|
|
7443
|
+
baseSha256: baseByPath.get(file.path)?.sha256,
|
|
7444
|
+
readFile: storageIo.readFile,
|
|
7445
|
+
readFileChunks: storageIo.readFileChunks
|
|
7446
|
+
});
|
|
7447
|
+
} catch (error) {
|
|
7448
|
+
directPushFailures.push({
|
|
7449
|
+
path: file.path,
|
|
7450
|
+
error: error instanceof Error ? error.message : String(error)
|
|
7451
|
+
});
|
|
7452
|
+
continue;
|
|
7453
|
+
}
|
|
7454
|
+
directPushNamespace = result.namespace ?? directPushNamespace;
|
|
7455
|
+
directPushedPaths.add(file.path);
|
|
7456
|
+
if (result.conflict) {
|
|
7457
|
+
directPushConflicts.push({
|
|
7458
|
+
path: result.conflict.path,
|
|
7459
|
+
reason: result.conflict.reason,
|
|
7460
|
+
...result.conflict.conflictPath ? { conflictPath: result.conflict.conflictPath } : {}
|
|
7461
|
+
});
|
|
7462
|
+
} else {
|
|
7463
|
+
if (result.applied) directPushAppliedUpserts += 1;
|
|
7464
|
+
if (result.skipped) directPushSkipped += 1;
|
|
7465
|
+
}
|
|
7466
|
+
}
|
|
7467
|
+
if (directPushFailures.length > 0) {
|
|
7468
|
+
throw new OfflineLargeFilePushError(directPushFailures);
|
|
7469
|
+
}
|
|
7033
7470
|
const changeset = await buildOfflineSyncChangeset({
|
|
7034
7471
|
root: options.memoryDir,
|
|
7035
|
-
sourceId:
|
|
7472
|
+
sourceId: localSourceId,
|
|
7036
7473
|
baseFiles,
|
|
7474
|
+
excludePaths: [...directPushedPaths],
|
|
7037
7475
|
includeTranscripts: options.includeTranscripts,
|
|
7038
7476
|
readFile: storageIo.readFile
|
|
7039
7477
|
});
|
|
7040
|
-
const
|
|
7041
|
-
const pushed = changeset.changes.length > 0 ? await pushOfflineChanges({
|
|
7478
|
+
const pushedInline = changeset.changes.length > 0 ? await pushOfflineChanges({
|
|
7042
7479
|
remoteUrl: options.remoteUrl,
|
|
7043
7480
|
token: options.token,
|
|
7044
7481
|
namespace: syncNamespace,
|
|
7045
7482
|
changeset
|
|
7046
7483
|
}) : null;
|
|
7484
|
+
const pushed = directPushedPaths.size > 0 || pushedInline ? {
|
|
7485
|
+
namespace: pushedInline?.namespace ?? directPushNamespace ?? syncNamespace ?? "",
|
|
7486
|
+
appliedUpserts: (pushedInline?.appliedUpserts ?? 0) + directPushAppliedUpserts,
|
|
7487
|
+
appliedDeletes: pushedInline?.appliedDeletes ?? 0,
|
|
7488
|
+
skipped: (pushedInline?.skipped ?? 0) + directPushSkipped,
|
|
7489
|
+
conflicts: [...directPushConflicts, ...pushedInline?.conflicts ?? []]
|
|
7490
|
+
} : null;
|
|
7047
7491
|
const remoteSnapshotMetadata = await fetchOfflineSnapshot({
|
|
7048
7492
|
remoteUrl: options.remoteUrl,
|
|
7049
7493
|
token: options.token,
|
|
@@ -7053,7 +7497,7 @@ async function runOfflineSyncOnce(options) {
|
|
|
7053
7497
|
});
|
|
7054
7498
|
const currentSnapshot = await buildOfflineSyncSnapshot({
|
|
7055
7499
|
root: options.memoryDir,
|
|
7056
|
-
sourceId:
|
|
7500
|
+
sourceId: localSourceId,
|
|
7057
7501
|
includeContent: false,
|
|
7058
7502
|
includeTranscripts: options.includeTranscripts,
|
|
7059
7503
|
readFile: storageIo.readFile
|
|
@@ -7071,7 +7515,7 @@ async function runOfflineSyncOnce(options) {
|
|
|
7071
7515
|
});
|
|
7072
7516
|
const applyCurrentSnapshot = directHydratedPaths.size > 0 ? await buildOfflineSyncSnapshot({
|
|
7073
7517
|
root: options.memoryDir,
|
|
7074
|
-
sourceId:
|
|
7518
|
+
sourceId: localSourceId,
|
|
7075
7519
|
includeContent: false,
|
|
7076
7520
|
includeTranscripts: options.includeTranscripts,
|
|
7077
7521
|
readFile: storageIo.readFile
|
|
@@ -10230,14 +10674,19 @@ if (argv1Base.endsWith("remnic.ts") || argv1Base.endsWith("remnic.js") || argv1B
|
|
|
10230
10674
|
}
|
|
10231
10675
|
export {
|
|
10232
10676
|
BENCHMARK_CATALOG,
|
|
10677
|
+
OFFLINE_SYNC_APPLY_MAX_REQUEST_BYTES,
|
|
10678
|
+
OFFLINE_SYNC_DIRECT_PUSH_MIN_BYTES,
|
|
10679
|
+
OFFLINE_SYNC_FILE_CONTENT_UPLOAD_CHUNK_BYTES,
|
|
10233
10680
|
TAXONOMY_RESOLVE_BOOLEAN_FLAGS,
|
|
10234
10681
|
TAXONOMY_RESOLVE_VALUE_FLAGS,
|
|
10235
10682
|
__benchDatasetTestHooks,
|
|
10236
10683
|
buildBenchRuntimeProfileRequest,
|
|
10237
10684
|
buildPackageBenchExecutionPlans,
|
|
10238
10685
|
buildQueryRecallRequest,
|
|
10686
|
+
chunkOfflineChangesetApplyBatches,
|
|
10239
10687
|
chunkOfflineFileContentBatches,
|
|
10240
10688
|
extractXrayRawArgs,
|
|
10689
|
+
formatOfflineLargeFilePushFailureMessage,
|
|
10241
10690
|
getBenchUsageText,
|
|
10242
10691
|
hasFlag,
|
|
10243
10692
|
main,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@remnic/cli",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.18",
|
|
4
4
|
"description": "CLI for Remnic memory — init, query, doctor, daemon management",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -26,8 +26,8 @@
|
|
|
26
26
|
},
|
|
27
27
|
"dependencies": {
|
|
28
28
|
"yaml": "^2.4.2",
|
|
29
|
-
"@remnic/
|
|
30
|
-
"@remnic/
|
|
29
|
+
"@remnic/plugin-pi": "^1.0.4",
|
|
30
|
+
"@remnic/core": "^1.1.24",
|
|
31
31
|
"@remnic/server": "^1.0.5"
|
|
32
32
|
},
|
|
33
33
|
"peerDependencies": {
|
|
@@ -76,12 +76,12 @@
|
|
|
76
76
|
"@remnic/bench": "1.0.1",
|
|
77
77
|
"@remnic/export-weclone": "1.0.1",
|
|
78
78
|
"@remnic/import-weclone": "1.0.1",
|
|
79
|
-
"@remnic/import-mem0": "0.1.0",
|
|
80
79
|
"@remnic/import-claude": "0.1.0",
|
|
81
80
|
"@remnic/import-chatgpt": "0.1.0",
|
|
81
|
+
"@remnic/import-gemini": "0.1.0",
|
|
82
|
+
"@remnic/import-mem0": "0.1.0",
|
|
82
83
|
"@remnic/import-lossless-claw": "0.1.1",
|
|
83
|
-
"@remnic/import-supermemory": "0.1.2"
|
|
84
|
-
"@remnic/import-gemini": "0.1.0"
|
|
84
|
+
"@remnic/import-supermemory": "0.1.2"
|
|
85
85
|
},
|
|
86
86
|
"license": "MIT",
|
|
87
87
|
"repository": {
|