@remnic/cli 1.0.22 → 1.0.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +848 -242
- package/package.json +6 -6
package/dist/index.js
CHANGED
|
@@ -91,15 +91,19 @@ import {
|
|
|
91
91
|
OFFLINE_SYNC_APPLY_MAX_BODY_BYTES,
|
|
92
92
|
OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES,
|
|
93
93
|
OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES,
|
|
94
|
+
OFFLINE_SYNC_SNAPSHOT_BASE_MAX_BODY_BYTES,
|
|
95
|
+
applyOfflineSyncFileContentChunk,
|
|
94
96
|
applyOfflineSyncSnapshot,
|
|
95
|
-
|
|
96
|
-
|
|
97
|
+
buildOfflineSyncChangesetFromSnapshot,
|
|
98
|
+
buildOfflineSyncSnapshotFromBase,
|
|
97
99
|
defaultOfflineSyncStatePath,
|
|
98
100
|
normalizeOfflineSyncSnapshot,
|
|
99
101
|
offlineSyncStateFromSnapshot,
|
|
100
102
|
readOfflineSyncFileContentChunk,
|
|
101
103
|
readOfflineSyncState,
|
|
104
|
+
shouldPreferIncomingOfflineRuntimeFile,
|
|
102
105
|
summarizeOfflineSyncPendingChanges,
|
|
106
|
+
summarizeOfflineSyncPendingFiles,
|
|
103
107
|
writeOfflineSyncState,
|
|
104
108
|
buildActionConfidenceInputFromOptions,
|
|
105
109
|
evaluateActionConfidence,
|
|
@@ -6643,29 +6647,128 @@ function offlineEndpoint(remoteUrl, pathname, params = {}) {
|
|
|
6643
6647
|
}
|
|
6644
6648
|
return url.toString();
|
|
6645
6649
|
}
|
|
6646
|
-
|
|
6647
|
-
|
|
6648
|
-
|
|
6649
|
-
|
|
6650
|
-
|
|
6651
|
-
|
|
6652
|
-
|
|
6650
|
+
var OFFLINE_SYNC_REQUEST_TIMEOUT_DEFAULT_MS = 15 * 6e4;
|
|
6651
|
+
function parseOfflineSyncRequestTimeoutMs(raw, fallback = OFFLINE_SYNC_REQUEST_TIMEOUT_DEFAULT_MS) {
|
|
6652
|
+
if (raw === void 0 || raw.trim().length === 0) return fallback;
|
|
6653
|
+
const parsed = Number(raw);
|
|
6654
|
+
if (!Number.isInteger(parsed) || parsed < 1e3) {
|
|
6655
|
+
throw new Error("REMNIC_OFFLINE_REQUEST_TIMEOUT_MS must be an integer >= 1000");
|
|
6656
|
+
}
|
|
6657
|
+
return parsed;
|
|
6658
|
+
}
|
|
6659
|
+
function formatOfflineRequestForError(url, init = {}) {
|
|
6660
|
+
const method = (init.method ?? "GET").toString().toUpperCase();
|
|
6661
|
+
try {
|
|
6662
|
+
const parsed = new URL(url);
|
|
6663
|
+
return `${method} ${parsed.pathname}${parsed.search}`;
|
|
6664
|
+
} catch {
|
|
6665
|
+
return `${method} ${url}`;
|
|
6666
|
+
}
|
|
6667
|
+
}
|
|
6668
|
+
function offlineRequestTimeoutMs() {
|
|
6669
|
+
return parseOfflineSyncRequestTimeoutMs(
|
|
6670
|
+
process.env.REMNIC_OFFLINE_REQUEST_TIMEOUT_MS ?? process.env.ENGRAM_OFFLINE_REQUEST_TIMEOUT_MS
|
|
6671
|
+
);
|
|
6672
|
+
}
|
|
6673
|
+
function offlineFetchHeaders(token, initHeaders, defaultContentType) {
|
|
6674
|
+
const headers = new Headers(initHeaders);
|
|
6675
|
+
headers.set("authorization", `Bearer ${token}`);
|
|
6676
|
+
if (defaultContentType && !headers.has("content-type")) {
|
|
6677
|
+
headers.set("content-type", defaultContentType);
|
|
6678
|
+
}
|
|
6679
|
+
return headers;
|
|
6680
|
+
}
|
|
6681
|
+
async function fetchOfflineWithResponse(url, token, init = {}, options = {}, consume) {
|
|
6682
|
+
const timeoutMs = offlineRequestTimeoutMs();
|
|
6683
|
+
const controller = new AbortController();
|
|
6684
|
+
const upstreamSignal = init.signal;
|
|
6685
|
+
const requestContext = formatOfflineRequestForError(url, init);
|
|
6686
|
+
let didTimeout = false;
|
|
6687
|
+
const timeout = setTimeout(() => {
|
|
6688
|
+
didTimeout = true;
|
|
6689
|
+
controller.abort();
|
|
6690
|
+
}, timeoutMs);
|
|
6691
|
+
const abortFromUpstream = () => controller.abort(upstreamSignal?.reason);
|
|
6692
|
+
if (upstreamSignal) {
|
|
6693
|
+
if (upstreamSignal.aborted) controller.abort(upstreamSignal.reason);
|
|
6694
|
+
else upstreamSignal.addEventListener("abort", abortFromUpstream, { once: true });
|
|
6695
|
+
}
|
|
6696
|
+
let response;
|
|
6697
|
+
try {
|
|
6698
|
+
response = await fetch(url, {
|
|
6699
|
+
...init,
|
|
6700
|
+
signal: controller.signal,
|
|
6701
|
+
headers: offlineFetchHeaders(token, init.headers, options.defaultContentType)
|
|
6702
|
+
});
|
|
6703
|
+
} catch (error) {
|
|
6704
|
+
clearTimeout(timeout);
|
|
6705
|
+
upstreamSignal?.removeEventListener("abort", abortFromUpstream);
|
|
6706
|
+
if (didTimeout) {
|
|
6707
|
+
throw new Error(`offline sync request timed out after ${timeoutMs}ms: ${requestContext}`);
|
|
6653
6708
|
}
|
|
6654
|
-
|
|
6655
|
-
|
|
6656
|
-
|
|
6657
|
-
|
|
6658
|
-
|
|
6659
|
-
|
|
6660
|
-
|
|
6709
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
6710
|
+
throw new Error(`offline sync request failed before response: ${requestContext} - ${message}`);
|
|
6711
|
+
}
|
|
6712
|
+
try {
|
|
6713
|
+
return await consume(response);
|
|
6714
|
+
} catch (error) {
|
|
6715
|
+
if (didTimeout) {
|
|
6716
|
+
throw new Error(`offline sync request timed out after ${timeoutMs}ms: ${requestContext}`);
|
|
6661
6717
|
}
|
|
6662
|
-
throw
|
|
6663
|
-
|
|
6664
|
-
);
|
|
6718
|
+
throw error;
|
|
6719
|
+
} finally {
|
|
6720
|
+
clearTimeout(timeout);
|
|
6721
|
+
upstreamSignal?.removeEventListener("abort", abortFromUpstream);
|
|
6665
6722
|
}
|
|
6666
|
-
|
|
6723
|
+
}
|
|
6724
|
+
async function throwOfflineResponseError(response, url, init, label = "offline sync request") {
|
|
6725
|
+
let detail = "";
|
|
6726
|
+
try {
|
|
6727
|
+
detail = await response.text();
|
|
6728
|
+
} catch {
|
|
6729
|
+
detail = "";
|
|
6730
|
+
}
|
|
6731
|
+
throw new Error(
|
|
6732
|
+
`${label} failed: ${formatOfflineRequestForError(url, init)} returned ${response.status} ${response.statusText}${detail ? ` - ${detail.slice(0, 500)}` : ""}`
|
|
6733
|
+
);
|
|
6734
|
+
}
|
|
6735
|
+
async function fetchOfflineJson(url, token, init = {}) {
|
|
6736
|
+
return fetchOfflineWithResponse(
|
|
6737
|
+
url,
|
|
6738
|
+
token,
|
|
6739
|
+
init,
|
|
6740
|
+
{ defaultContentType: init.body !== void 0 ? "application/json" : void 0 },
|
|
6741
|
+
async (response) => {
|
|
6742
|
+
if (!response.ok) {
|
|
6743
|
+
await throwOfflineResponseError(response, url, init);
|
|
6744
|
+
}
|
|
6745
|
+
return await response.json();
|
|
6746
|
+
}
|
|
6747
|
+
);
|
|
6667
6748
|
}
|
|
6668
6749
|
async function fetchOfflineSnapshot(args) {
|
|
6750
|
+
if (args.includeContent === false && args.baseFiles && args.baseFiles.length > 0) {
|
|
6751
|
+
const postBody = offlineSnapshotBasePostBody({
|
|
6752
|
+
namespace: args.namespace,
|
|
6753
|
+
includeTranscripts: args.includeTranscripts,
|
|
6754
|
+
baseFiles: args.baseFiles,
|
|
6755
|
+
baseCapturedAt: args.baseCapturedAt
|
|
6756
|
+
});
|
|
6757
|
+
if (offlineSnapshotBasePostBodyFits(postBody)) {
|
|
6758
|
+
try {
|
|
6759
|
+
return await fetchOfflineJson(
|
|
6760
|
+
offlineEndpoint(args.remoteUrl, "/remnic/v1/offline-sync/snapshot"),
|
|
6761
|
+
args.token,
|
|
6762
|
+
{
|
|
6763
|
+
method: "POST",
|
|
6764
|
+
body: postBody
|
|
6765
|
+
}
|
|
6766
|
+
);
|
|
6767
|
+
} catch (error) {
|
|
6768
|
+
if (!isOfflineSnapshotPostFallbackError(error)) throw error;
|
|
6769
|
+
}
|
|
6770
|
+
}
|
|
6771
|
+
}
|
|
6669
6772
|
return fetchOfflineJson(
|
|
6670
6773
|
offlineEndpoint(args.remoteUrl, "/remnic/v1/offline-sync/snapshot", {
|
|
6671
6774
|
namespace: args.namespace,
|
|
@@ -6675,6 +6778,22 @@ async function fetchOfflineSnapshot(args) {
|
|
|
6675
6778
|
args.token
|
|
6676
6779
|
);
|
|
6677
6780
|
}
|
|
6781
|
+
function offlineSnapshotBasePostBody(args) {
|
|
6782
|
+
return JSON.stringify({
|
|
6783
|
+
namespace: args.namespace,
|
|
6784
|
+
includeTranscripts: args.includeTranscripts,
|
|
6785
|
+
includeContent: false,
|
|
6786
|
+
baseFiles: args.baseFiles,
|
|
6787
|
+
...args.baseCapturedAt ? { baseCapturedAt: args.baseCapturedAt.toISOString() } : {}
|
|
6788
|
+
});
|
|
6789
|
+
}
|
|
6790
|
+
function offlineSnapshotBasePostBodyFits(body) {
|
|
6791
|
+
return Buffer.byteLength(body, "utf-8") <= OFFLINE_SYNC_SNAPSHOT_BASE_MAX_BODY_BYTES;
|
|
6792
|
+
}
|
|
6793
|
+
function isOfflineSnapshotPostFallbackError(error) {
|
|
6794
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
6795
|
+
return /offline-sync\/snapshot\b.* returned (404|405|413)\b/.test(message);
|
|
6796
|
+
}
|
|
6678
6797
|
async function fetchOfflineFiles(args) {
|
|
6679
6798
|
return fetchOfflineJson(
|
|
6680
6799
|
offlineEndpoint(args.remoteUrl, "/remnic/v1/offline-sync/files"),
|
|
@@ -6702,6 +6821,28 @@ var OFFLINE_SYNC_DIRECT_PUSH_MIN_BYTES = Math.min(
|
|
|
6702
6821
|
OFFLINE_SYNC_INLINE_CONTENT_MAX_BYTES
|
|
6703
6822
|
);
|
|
6704
6823
|
var OFFLINE_SYNC_DIRECT_HYDRATE_MIN_BYTES = OFFLINE_SYNC_DIRECT_PUSH_MIN_BYTES;
|
|
6824
|
+
var OFFLINE_SYNC_CHANGESET_RETRY_MAX = 1024;
|
|
6825
|
+
var OfflineRemoteFileChangedError = class extends Error {
|
|
6826
|
+
path;
|
|
6827
|
+
constructor(path12) {
|
|
6828
|
+
super(`remote file changed while fetching offline content: ${path12}`);
|
|
6829
|
+
this.name = "OfflineRemoteFileChangedError";
|
|
6830
|
+
this.path = path12;
|
|
6831
|
+
}
|
|
6832
|
+
};
|
|
6833
|
+
function isOfflineRemoteFileChangedError(error) {
|
|
6834
|
+
return error instanceof OfflineRemoteFileChangedError || error instanceof Error && error.message.startsWith("remote file changed while fetching offline content: ");
|
|
6835
|
+
}
|
|
6836
|
+
function isOfflineLocalFileChangedError(error) {
|
|
6837
|
+
return error instanceof Error && error.message.startsWith("local file changed while pushing offline content: ");
|
|
6838
|
+
}
|
|
6839
|
+
function offlineChangesetFileChangedPath(error) {
|
|
6840
|
+
if (!(error instanceof Error)) return null;
|
|
6841
|
+
const prefix = "offline sync file changed while building changeset: ";
|
|
6842
|
+
if (!error.message.startsWith(prefix)) return null;
|
|
6843
|
+
const relPath = error.message.slice(prefix.length).trim();
|
|
6844
|
+
return relPath.length > 0 ? relPath : null;
|
|
6845
|
+
}
|
|
6705
6846
|
function parseOfflineHeaderNumber(headers, name) {
|
|
6706
6847
|
const raw = headers.get(name);
|
|
6707
6848
|
if (raw === null) throw new Error(`offline file content response omitted ${name}`);
|
|
@@ -6712,86 +6853,77 @@ function parseOfflineHeaderNumber(headers, name) {
|
|
|
6712
6853
|
return parsed;
|
|
6713
6854
|
}
|
|
6714
6855
|
async function fetchOfflineFileContentChunk(args) {
|
|
6715
|
-
const
|
|
6716
|
-
|
|
6717
|
-
|
|
6718
|
-
|
|
6719
|
-
|
|
6720
|
-
|
|
6721
|
-
|
|
6722
|
-
|
|
6723
|
-
|
|
6724
|
-
|
|
6725
|
-
|
|
6726
|
-
|
|
6727
|
-
|
|
6728
|
-
|
|
6729
|
-
|
|
6856
|
+
const url = offlineEndpoint(args.remoteUrl, "/remnic/v1/offline-sync/file-content");
|
|
6857
|
+
const init = {
|
|
6858
|
+
method: "POST",
|
|
6859
|
+
body: JSON.stringify({
|
|
6860
|
+
namespace: args.namespace,
|
|
6861
|
+
includeTranscripts: args.includeTranscripts,
|
|
6862
|
+
path: args.path,
|
|
6863
|
+
offset: args.offset,
|
|
6864
|
+
length: args.length
|
|
6865
|
+
})
|
|
6866
|
+
};
|
|
6867
|
+
return fetchOfflineWithResponse(
|
|
6868
|
+
url,
|
|
6869
|
+
args.token,
|
|
6870
|
+
init,
|
|
6871
|
+
{ defaultContentType: "application/json" },
|
|
6872
|
+
async (response) => {
|
|
6873
|
+
if (!response.ok) {
|
|
6874
|
+
await throwOfflineResponseError(response, url, init, "offline sync file-content request");
|
|
6875
|
+
}
|
|
6876
|
+
const encodedPath = response.headers.get("x-remnic-file-path");
|
|
6877
|
+
const relPath = encodedPath ? decodeURIComponent(encodedPath) : args.path;
|
|
6878
|
+
const content = Buffer.from(await response.arrayBuffer());
|
|
6879
|
+
const chunkBytes = parseOfflineHeaderNumber(response.headers, "x-remnic-chunk-bytes");
|
|
6880
|
+
const sha256 = response.headers.get("x-remnic-file-sha256") ?? void 0;
|
|
6881
|
+
if (content.length !== chunkBytes) {
|
|
6882
|
+
throw new Error(`offline file content response length mismatch for ${relPath}`);
|
|
6883
|
+
}
|
|
6884
|
+
return {
|
|
6885
|
+
path: relPath,
|
|
6886
|
+
...sha256 ? { sha256 } : {},
|
|
6887
|
+
bytes: parseOfflineHeaderNumber(response.headers, "x-remnic-file-bytes"),
|
|
6888
|
+
mtimeMs: parseOfflineHeaderNumber(response.headers, "x-remnic-file-mtime-ms"),
|
|
6889
|
+
offset: parseOfflineHeaderNumber(response.headers, "x-remnic-chunk-offset"),
|
|
6890
|
+
chunkBytes,
|
|
6891
|
+
content
|
|
6892
|
+
};
|
|
6730
6893
|
}
|
|
6731
6894
|
);
|
|
6732
|
-
if (!response.ok) {
|
|
6733
|
-
let detail = "";
|
|
6734
|
-
try {
|
|
6735
|
-
detail = await response.text();
|
|
6736
|
-
} catch {
|
|
6737
|
-
detail = "";
|
|
6738
|
-
}
|
|
6739
|
-
throw new Error(
|
|
6740
|
-
`offline sync file-content request failed: ${response.status} ${response.statusText}${detail ? ` - ${detail.slice(0, 500)}` : ""}`
|
|
6741
|
-
);
|
|
6742
|
-
}
|
|
6743
|
-
const encodedPath = response.headers.get("x-remnic-file-path");
|
|
6744
|
-
const relPath = encodedPath ? decodeURIComponent(encodedPath) : args.path;
|
|
6745
|
-
const content = Buffer.from(await response.arrayBuffer());
|
|
6746
|
-
const chunkBytes = parseOfflineHeaderNumber(response.headers, "x-remnic-chunk-bytes");
|
|
6747
|
-
const sha256 = response.headers.get("x-remnic-file-sha256") ?? void 0;
|
|
6748
|
-
if (content.length !== chunkBytes) {
|
|
6749
|
-
throw new Error(`offline file content response length mismatch for ${relPath}`);
|
|
6750
|
-
}
|
|
6751
|
-
return {
|
|
6752
|
-
path: relPath,
|
|
6753
|
-
...sha256 ? { sha256 } : {},
|
|
6754
|
-
bytes: parseOfflineHeaderNumber(response.headers, "x-remnic-file-bytes"),
|
|
6755
|
-
mtimeMs: parseOfflineHeaderNumber(response.headers, "x-remnic-file-mtime-ms"),
|
|
6756
|
-
offset: parseOfflineHeaderNumber(response.headers, "x-remnic-chunk-offset"),
|
|
6757
|
-
chunkBytes,
|
|
6758
|
-
content
|
|
6759
|
-
};
|
|
6760
6895
|
}
|
|
6761
6896
|
async function postOfflineFileContentChunk(args) {
|
|
6762
|
-
const
|
|
6763
|
-
|
|
6764
|
-
|
|
6765
|
-
|
|
6766
|
-
|
|
6767
|
-
|
|
6768
|
-
|
|
6769
|
-
|
|
6770
|
-
|
|
6771
|
-
|
|
6772
|
-
|
|
6773
|
-
|
|
6774
|
-
|
|
6775
|
-
|
|
6776
|
-
|
|
6777
|
-
|
|
6778
|
-
|
|
6779
|
-
|
|
6780
|
-
|
|
6897
|
+
const url = offlineEndpoint(args.remoteUrl, "/remnic/v1/offline-sync/apply-file-content", {
|
|
6898
|
+
namespace: args.namespace
|
|
6899
|
+
});
|
|
6900
|
+
const init = {
|
|
6901
|
+
method: "POST",
|
|
6902
|
+
headers: {
|
|
6903
|
+
"content-type": "application/octet-stream",
|
|
6904
|
+
"x-remnic-include-transcripts": args.includeTranscripts ? "true" : "false",
|
|
6905
|
+
"x-remnic-source-id": encodeURIComponent(args.sourceId),
|
|
6906
|
+
"x-remnic-file-path": encodeURIComponent(args.file.path),
|
|
6907
|
+
"x-remnic-file-sha256": args.file.sha256,
|
|
6908
|
+
"x-remnic-file-bytes": String(args.file.bytes),
|
|
6909
|
+
"x-remnic-file-mtime-ms": String(args.file.mtimeMs),
|
|
6910
|
+
"x-remnic-chunk-offset": String(args.offset),
|
|
6911
|
+
...args.baseSha256 ? { "x-remnic-base-sha256": args.baseSha256 } : {}
|
|
6912
|
+
},
|
|
6913
|
+
body: new Blob([new Uint8Array(args.content)])
|
|
6914
|
+
};
|
|
6915
|
+
return fetchOfflineWithResponse(
|
|
6916
|
+
url,
|
|
6917
|
+
args.token,
|
|
6918
|
+
init,
|
|
6919
|
+
{},
|
|
6920
|
+
async (response) => {
|
|
6921
|
+
if (!response.ok) {
|
|
6922
|
+
await throwOfflineResponseError(response, url, init, "offline sync apply-file-content request");
|
|
6923
|
+
}
|
|
6924
|
+
return await response.json();
|
|
6781
6925
|
}
|
|
6782
6926
|
);
|
|
6783
|
-
if (!response.ok) {
|
|
6784
|
-
let detail = "";
|
|
6785
|
-
try {
|
|
6786
|
-
detail = await response.text();
|
|
6787
|
-
} catch {
|
|
6788
|
-
detail = "";
|
|
6789
|
-
}
|
|
6790
|
-
throw new Error(
|
|
6791
|
-
`offline sync apply-file-content request failed: ${response.status} ${response.statusText}${detail ? ` - ${detail.slice(0, 500)}` : ""}`
|
|
6792
|
-
);
|
|
6793
|
-
}
|
|
6794
|
-
return await response.json();
|
|
6795
6927
|
}
|
|
6796
6928
|
function resolvedOfflineSnapshotNamespace(snapshot, requestedNamespace) {
|
|
6797
6929
|
const resolved = typeof snapshot.namespace === "string" && snapshot.namespace.trim().length > 0 ? snapshot.namespace.trim() : void 0;
|
|
@@ -6831,10 +6963,29 @@ function offlineFileStateMap(files) {
|
|
|
6831
6963
|
function offlineSnapshotContentFilesForApply(options) {
|
|
6832
6964
|
const base = offlineFileStateMap(options.baseFiles);
|
|
6833
6965
|
const current = options.currentFiles ? offlineFileStateMap(options.currentFiles) : null;
|
|
6966
|
+
const conflictContentMaxBytes = options.conflictContentMaxBytes ?? Number.POSITIVE_INFINITY;
|
|
6967
|
+
const deferredPaths = new Set(options.deferredPaths ?? []);
|
|
6834
6968
|
const files = [];
|
|
6835
6969
|
for (const incoming of options.snapshot.files) {
|
|
6836
|
-
if (
|
|
6837
|
-
|
|
6970
|
+
if (deferredPaths.has(incoming.path)) continue;
|
|
6971
|
+
const baseEntry = base.get(incoming.path);
|
|
6972
|
+
const currentEntry = current?.get(incoming.path);
|
|
6973
|
+
if (currentEntry?.sha256 === incoming.sha256) continue;
|
|
6974
|
+
if (currentEntry && baseEntry && incoming.sha256 === baseEntry.sha256) continue;
|
|
6975
|
+
if (shouldPreferIncomingOfflineRuntimeFile(incoming.path)) {
|
|
6976
|
+
files.push(incoming);
|
|
6977
|
+
continue;
|
|
6978
|
+
}
|
|
6979
|
+
if (!currentEntry && baseEntry && incoming.sha256 === baseEntry.sha256) continue;
|
|
6980
|
+
if (!currentEntry && !baseEntry) {
|
|
6981
|
+
files.push(incoming);
|
|
6982
|
+
continue;
|
|
6983
|
+
}
|
|
6984
|
+
if (baseEntry && currentEntry && currentEntry.sha256 === baseEntry.sha256) {
|
|
6985
|
+
files.push(incoming);
|
|
6986
|
+
continue;
|
|
6987
|
+
}
|
|
6988
|
+
if (incoming.bytes > conflictContentMaxBytes) continue;
|
|
6838
6989
|
files.push(incoming);
|
|
6839
6990
|
}
|
|
6840
6991
|
return files.sort((left, right) => left.path.localeCompare(right.path));
|
|
@@ -6842,15 +6993,24 @@ function offlineSnapshotContentFilesForApply(options) {
|
|
|
6842
6993
|
function shouldDirectHydrateOfflineFile(options) {
|
|
6843
6994
|
if (options.incoming.bytes < OFFLINE_SYNC_DIRECT_HYDRATE_MIN_BYTES) return false;
|
|
6844
6995
|
if (options.current?.sha256 === options.incoming.sha256) return false;
|
|
6996
|
+
if (shouldPreferIncomingOfflineRuntimeFile(options.incoming.path)) return true;
|
|
6845
6997
|
if (options.current && options.base && options.current.sha256 === options.base.sha256) {
|
|
6846
6998
|
return true;
|
|
6847
6999
|
}
|
|
6848
7000
|
return !options.current && !options.base;
|
|
6849
7001
|
}
|
|
7002
|
+
function offlinePartialHydrationForPaths(options) {
|
|
7003
|
+
const hydratedPaths = new Set(options.hydratedPaths);
|
|
7004
|
+
return {
|
|
7005
|
+
hydratedFiles: options.files.filter((file) => hydratedPaths.has(file.path)),
|
|
7006
|
+
remoteDeferredPaths: [...options.deferredPaths]
|
|
7007
|
+
};
|
|
7008
|
+
}
|
|
6850
7009
|
function offlineDirectPushFiles(options) {
|
|
6851
7010
|
const base = offlineFileStateMap(options.baseFiles);
|
|
6852
7011
|
return options.currentFiles.filter((current) => {
|
|
6853
7012
|
if (current.bytes < OFFLINE_SYNC_DIRECT_PUSH_MIN_BYTES) return false;
|
|
7013
|
+
if (shouldPreferIncomingOfflineRuntimeFile(current.path)) return false;
|
|
6854
7014
|
return current.sha256 !== base.get(current.path)?.sha256;
|
|
6855
7015
|
}).sort((left, right) => right.bytes - left.bytes || left.path.localeCompare(right.path));
|
|
6856
7016
|
}
|
|
@@ -6870,6 +7030,9 @@ async function pushOfflineFileContent(args) {
|
|
|
6870
7030
|
}
|
|
6871
7031
|
let offset = 0;
|
|
6872
7032
|
let finalResult = null;
|
|
7033
|
+
let remoteSatisfiedResult = null;
|
|
7034
|
+
const hash = createHash("sha256");
|
|
7035
|
+
let bytes = 0;
|
|
6873
7036
|
while (offset < args.file.bytes || args.file.bytes === 0 && offset === 0) {
|
|
6874
7037
|
const chunk = await readOfflineSyncFileContentChunk({
|
|
6875
7038
|
root: args.memoryDir,
|
|
@@ -6888,23 +7051,36 @@ async function pushOfflineFileContent(args) {
|
|
|
6888
7051
|
if (chunk.chunkBytes === 0 && args.file.bytes > 0) {
|
|
6889
7052
|
throw new Error(`local offline content chunk was empty before EOF: ${args.file.path}`);
|
|
6890
7053
|
}
|
|
6891
|
-
|
|
6892
|
-
|
|
6893
|
-
|
|
6894
|
-
|
|
6895
|
-
|
|
6896
|
-
|
|
6897
|
-
|
|
6898
|
-
|
|
6899
|
-
|
|
6900
|
-
|
|
6901
|
-
|
|
6902
|
-
|
|
6903
|
-
|
|
7054
|
+
hash.update(chunk.content);
|
|
7055
|
+
bytes += chunk.chunkBytes;
|
|
7056
|
+
if (!remoteSatisfiedResult) {
|
|
7057
|
+
finalResult = await postOfflineFileContentChunk({
|
|
7058
|
+
remoteUrl: args.remoteUrl,
|
|
7059
|
+
token: args.token,
|
|
7060
|
+
namespace: args.namespace,
|
|
7061
|
+
includeTranscripts: args.includeTranscripts,
|
|
7062
|
+
sourceId: args.sourceId,
|
|
7063
|
+
file: args.file,
|
|
7064
|
+
baseSha256: args.baseSha256,
|
|
7065
|
+
offset,
|
|
7066
|
+
content: chunk.content
|
|
7067
|
+
});
|
|
7068
|
+
if (finalResult.conflict) {
|
|
7069
|
+
return finalResult;
|
|
7070
|
+
}
|
|
7071
|
+
if (finalResult.done && finalResult.skipped) {
|
|
7072
|
+
remoteSatisfiedResult = finalResult;
|
|
7073
|
+
}
|
|
6904
7074
|
}
|
|
6905
7075
|
offset += chunk.chunkBytes;
|
|
6906
7076
|
if (args.file.bytes === 0) break;
|
|
6907
7077
|
}
|
|
7078
|
+
if (hash.digest("hex") !== args.file.sha256 || bytes !== args.file.bytes) {
|
|
7079
|
+
throw new Error(`local file changed while pushing offline content: ${args.file.path}`);
|
|
7080
|
+
}
|
|
7081
|
+
if (remoteSatisfiedResult) {
|
|
7082
|
+
return remoteSatisfiedResult;
|
|
7083
|
+
}
|
|
6908
7084
|
if (!finalResult?.done) {
|
|
6909
7085
|
throw new Error(`offline sync large-file push did not finish for ${args.file.path}`);
|
|
6910
7086
|
}
|
|
@@ -6926,6 +7102,7 @@ async function pushOfflineFileContentFromChunkReader(args) {
|
|
|
6926
7102
|
let offset = 0;
|
|
6927
7103
|
let pending = null;
|
|
6928
7104
|
let finalResult = null;
|
|
7105
|
+
let remoteSatisfiedResult = null;
|
|
6929
7106
|
for await (const rawChunk of chunks) {
|
|
6930
7107
|
const chunk = Buffer.from(rawChunk);
|
|
6931
7108
|
if (chunk.length === 0) continue;
|
|
@@ -6934,19 +7111,24 @@ async function pushOfflineFileContentFromChunkReader(args) {
|
|
|
6934
7111
|
}
|
|
6935
7112
|
if (pending) {
|
|
6936
7113
|
hash.update(pending);
|
|
6937
|
-
|
|
6938
|
-
|
|
6939
|
-
|
|
6940
|
-
|
|
6941
|
-
|
|
6942
|
-
|
|
6943
|
-
|
|
6944
|
-
|
|
6945
|
-
|
|
6946
|
-
|
|
6947
|
-
|
|
6948
|
-
|
|
6949
|
-
|
|
7114
|
+
if (!remoteSatisfiedResult) {
|
|
7115
|
+
finalResult = await postOfflineFileContentChunk({
|
|
7116
|
+
remoteUrl: args.remoteUrl,
|
|
7117
|
+
token: args.token,
|
|
7118
|
+
namespace: args.namespace,
|
|
7119
|
+
includeTranscripts: args.includeTranscripts,
|
|
7120
|
+
sourceId: args.sourceId,
|
|
7121
|
+
file: args.file,
|
|
7122
|
+
baseSha256: args.baseSha256,
|
|
7123
|
+
offset,
|
|
7124
|
+
content: pending
|
|
7125
|
+
});
|
|
7126
|
+
if (finalResult.conflict) {
|
|
7127
|
+
return finalResult;
|
|
7128
|
+
}
|
|
7129
|
+
if (finalResult.done && finalResult.skipped) {
|
|
7130
|
+
remoteSatisfiedResult = finalResult;
|
|
7131
|
+
}
|
|
6950
7132
|
}
|
|
6951
7133
|
offset += pending.length;
|
|
6952
7134
|
}
|
|
@@ -6958,6 +7140,9 @@ async function pushOfflineFileContentFromChunkReader(args) {
|
|
|
6958
7140
|
if (digest !== args.file.sha256 || finalBytes !== args.file.bytes) {
|
|
6959
7141
|
throw new Error(`local file changed while pushing offline content: ${args.file.path}`);
|
|
6960
7142
|
}
|
|
7143
|
+
if (remoteSatisfiedResult) {
|
|
7144
|
+
return remoteSatisfiedResult;
|
|
7145
|
+
}
|
|
6961
7146
|
if (pending) {
|
|
6962
7147
|
finalResult = await postOfflineFileContentChunk({
|
|
6963
7148
|
remoteUrl: args.remoteUrl,
|
|
@@ -6973,6 +7158,9 @@ async function pushOfflineFileContentFromChunkReader(args) {
|
|
|
6973
7158
|
if (finalResult.conflict) {
|
|
6974
7159
|
return finalResult;
|
|
6975
7160
|
}
|
|
7161
|
+
if (finalResult.done && finalResult.skipped) {
|
|
7162
|
+
return finalResult;
|
|
7163
|
+
}
|
|
6976
7164
|
} else if (args.file.bytes === 0) {
|
|
6977
7165
|
finalResult = await postOfflineFileContentChunk({
|
|
6978
7166
|
remoteUrl: args.remoteUrl,
|
|
@@ -6991,11 +7179,10 @@ async function pushOfflineFileContentFromChunkReader(args) {
|
|
|
6991
7179
|
}
|
|
6992
7180
|
return finalResult;
|
|
6993
7181
|
}
|
|
6994
|
-
async function
|
|
6995
|
-
const chunks = [];
|
|
6996
|
-
const hash = createHash("sha256");
|
|
7182
|
+
async function hydrateOfflineFileContent(args) {
|
|
6997
7183
|
let offset = 0;
|
|
6998
|
-
|
|
7184
|
+
let finalResult = null;
|
|
7185
|
+
while (offset < args.expected.bytes || args.expected.bytes === 0 && offset === 0) {
|
|
6999
7186
|
const chunk = await fetchOfflineFileContentChunk({
|
|
7000
7187
|
remoteUrl: args.remoteUrl,
|
|
7001
7188
|
token: args.token,
|
|
@@ -7005,54 +7192,92 @@ async function fetchOfflineFileContent(args) {
|
|
|
7005
7192
|
offset,
|
|
7006
7193
|
length: Math.min(
|
|
7007
7194
|
OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES,
|
|
7008
|
-
args.expected.bytes - offset
|
|
7195
|
+
Math.max(1, args.expected.bytes - offset)
|
|
7009
7196
|
)
|
|
7010
7197
|
});
|
|
7011
7198
|
if (chunk.path !== args.expected.path || chunk.sha256 !== void 0 && chunk.sha256 !== args.expected.sha256 || chunk.bytes !== args.expected.bytes || chunk.mtimeMs !== args.expected.mtimeMs || chunk.offset !== offset || chunk.chunkBytes !== chunk.content.length) {
|
|
7012
|
-
throw new
|
|
7199
|
+
throw new OfflineRemoteFileChangedError(args.expected.path);
|
|
7013
7200
|
}
|
|
7014
|
-
if (chunk.chunkBytes === 0) {
|
|
7201
|
+
if (chunk.chunkBytes === 0 && args.expected.bytes > 0) {
|
|
7015
7202
|
throw new Error(`remote offline content chunk was empty before EOF: ${args.expected.path}`);
|
|
7016
7203
|
}
|
|
7017
|
-
|
|
7018
|
-
|
|
7204
|
+
finalResult = await applyOfflineSyncFileContentChunk({
|
|
7205
|
+
root: args.memoryDir,
|
|
7206
|
+
sourceId: args.sourceId,
|
|
7207
|
+
path: args.expected.path,
|
|
7208
|
+
sha256: args.expected.sha256,
|
|
7209
|
+
bytes: args.expected.bytes,
|
|
7210
|
+
mtimeMs: args.expected.mtimeMs,
|
|
7211
|
+
offset,
|
|
7212
|
+
content: chunk.content,
|
|
7213
|
+
...args.baseSha256 ? { baseSha256: args.baseSha256 } : {},
|
|
7214
|
+
includeTranscripts: args.includeTranscripts,
|
|
7215
|
+
readFile: args.readFile,
|
|
7216
|
+
readFileDigest: args.readFileDigest,
|
|
7217
|
+
writeFile: args.writeFile,
|
|
7218
|
+
writeStagingFile: args.writeStagingFile,
|
|
7219
|
+
writeFileChunks: args.writeFileChunks
|
|
7220
|
+
});
|
|
7221
|
+
if (finalResult.conflict) {
|
|
7222
|
+
return finalResult;
|
|
7223
|
+
}
|
|
7224
|
+
if (finalResult.done && finalResult.skipped) {
|
|
7225
|
+
return finalResult;
|
|
7226
|
+
}
|
|
7019
7227
|
offset += chunk.chunkBytes;
|
|
7228
|
+
if (args.expected.bytes === 0) break;
|
|
7020
7229
|
}
|
|
7021
|
-
|
|
7022
|
-
|
|
7023
|
-
if (digest !== args.expected.sha256 || content.length !== args.expected.bytes) {
|
|
7024
|
-
throw new Error(`remote offline content checksum mismatch for ${args.expected.path}`);
|
|
7230
|
+
if (!finalResult?.done) {
|
|
7231
|
+
throw new Error(`offline sync large-file hydrate did not finish for ${args.expected.path}`);
|
|
7025
7232
|
}
|
|
7026
|
-
return
|
|
7233
|
+
return finalResult;
|
|
7027
7234
|
}
|
|
7028
7235
|
async function directHydrateLargeOfflineFiles(args) {
|
|
7029
|
-
if (!args.
|
|
7236
|
+
if (!args.readFile || !args.writeFile || !args.writeStagingFile || !args.writeFileChunks) {
|
|
7237
|
+
return { hydratedPaths: /* @__PURE__ */ new Set(), deferredPaths: /* @__PURE__ */ new Set() };
|
|
7238
|
+
}
|
|
7030
7239
|
const snapshot = normalizeOfflineSyncSnapshot(args.snapshot);
|
|
7031
7240
|
const base = offlineFileStateMap(args.baseFiles);
|
|
7032
7241
|
const current = offlineFileStateMap(args.currentFiles);
|
|
7033
|
-
const
|
|
7242
|
+
const hydratedPaths = args.hydrationProgress?.hydratedPaths ?? /* @__PURE__ */ new Set();
|
|
7243
|
+
const deferredPaths = args.hydrationProgress?.deferredPaths ?? /* @__PURE__ */ new Set();
|
|
7034
7244
|
const candidates = snapshot.files.filter((incoming) => shouldDirectHydrateOfflineFile({
|
|
7035
7245
|
incoming,
|
|
7036
7246
|
base: base.get(incoming.path),
|
|
7037
7247
|
current: current.get(incoming.path)
|
|
7038
7248
|
})).sort((left, right) => right.bytes - left.bytes || left.path.localeCompare(right.path));
|
|
7039
7249
|
for (const incoming of candidates) {
|
|
7040
|
-
|
|
7041
|
-
|
|
7042
|
-
|
|
7043
|
-
|
|
7044
|
-
|
|
7045
|
-
|
|
7046
|
-
|
|
7047
|
-
|
|
7048
|
-
|
|
7049
|
-
|
|
7050
|
-
|
|
7051
|
-
|
|
7052
|
-
|
|
7053
|
-
|
|
7250
|
+
let result;
|
|
7251
|
+
try {
|
|
7252
|
+
result = await hydrateOfflineFileContent({
|
|
7253
|
+
remoteUrl: args.remoteUrl,
|
|
7254
|
+
token: args.token,
|
|
7255
|
+
namespace: args.namespace,
|
|
7256
|
+
includeTranscripts: args.includeTranscripts,
|
|
7257
|
+
memoryDir: args.memoryDir,
|
|
7258
|
+
sourceId: "remote",
|
|
7259
|
+
expected: incoming,
|
|
7260
|
+
baseSha256: base.get(incoming.path)?.sha256,
|
|
7261
|
+
readFile: args.readFile,
|
|
7262
|
+
readFileDigest: args.readFileDigest,
|
|
7263
|
+
writeFile: args.writeFile,
|
|
7264
|
+
writeStagingFile: args.writeStagingFile,
|
|
7265
|
+
writeFileChunks: args.writeFileChunks
|
|
7266
|
+
});
|
|
7267
|
+
} catch (error) {
|
|
7268
|
+
if (!isOfflineRemoteFileChangedError(error)) throw error;
|
|
7269
|
+
deferredPaths.add(incoming.path);
|
|
7270
|
+
continue;
|
|
7271
|
+
}
|
|
7272
|
+
if (result.conflict) {
|
|
7273
|
+
deferredPaths.add(result.conflict.path);
|
|
7274
|
+
continue;
|
|
7275
|
+
}
|
|
7276
|
+
if (result.applied || result.skipped) {
|
|
7277
|
+
hydratedPaths.add(incoming.path);
|
|
7278
|
+
}
|
|
7054
7279
|
}
|
|
7055
|
-
return
|
|
7280
|
+
return { hydratedPaths, deferredPaths };
|
|
7056
7281
|
}
|
|
7057
7282
|
function chunkOfflineFileContentBatches(files) {
|
|
7058
7283
|
const chunks = [];
|
|
@@ -7076,7 +7301,7 @@ function chunkOfflineFileContentBatches(files) {
|
|
|
7076
7301
|
}
|
|
7077
7302
|
function isOfflineFilesUnsupportedError(error) {
|
|
7078
7303
|
const message = error instanceof Error ? error.message : String(error);
|
|
7079
|
-
return /offline sync request failed: 404\b/.test(message);
|
|
7304
|
+
return /offline sync request failed: .* returned 404\b/.test(message);
|
|
7080
7305
|
}
|
|
7081
7306
|
function isMissingOfflineContentError(error) {
|
|
7082
7307
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -7087,7 +7312,9 @@ async function hydrateOfflineSnapshotContent(args) {
|
|
|
7087
7312
|
const neededFiles = offlineSnapshotContentFilesForApply({
|
|
7088
7313
|
snapshot,
|
|
7089
7314
|
baseFiles: args.baseFiles,
|
|
7090
|
-
currentFiles: args.currentFiles
|
|
7315
|
+
currentFiles: args.currentFiles,
|
|
7316
|
+
conflictContentMaxBytes: OFFLINE_SYNC_FILES_CONTENT_MAX_BATCH_BYTES,
|
|
7317
|
+
deferredPaths: args.deferredPaths
|
|
7091
7318
|
});
|
|
7092
7319
|
if (neededFiles.length === 0) return { ...args.snapshot, files: snapshot.files };
|
|
7093
7320
|
const expectedByPath = new Map(snapshot.files.map((file) => [file.path, file]));
|
|
@@ -7180,7 +7407,8 @@ async function postOfflineChangesBatch(args) {
|
|
|
7180
7407
|
method: "POST",
|
|
7181
7408
|
body: JSON.stringify({
|
|
7182
7409
|
namespace: args.namespace,
|
|
7183
|
-
changeset: args.changeset
|
|
7410
|
+
changeset: args.changeset,
|
|
7411
|
+
returnCurrentFiles: false
|
|
7184
7412
|
})
|
|
7185
7413
|
}
|
|
7186
7414
|
);
|
|
@@ -7201,6 +7429,7 @@ async function pushOfflineChanges(args) {
|
|
|
7201
7429
|
appliedDeletes += result.appliedDeletes;
|
|
7202
7430
|
skipped += result.skipped;
|
|
7203
7431
|
conflicts.push(...result.conflicts);
|
|
7432
|
+
args.onBatchApplied?.({ changeset, result });
|
|
7204
7433
|
}
|
|
7205
7434
|
return {
|
|
7206
7435
|
namespace,
|
|
@@ -7246,6 +7475,24 @@ async function createOfflineStorageIo(memoryDir) {
|
|
|
7246
7475
|
}
|
|
7247
7476
|
return {
|
|
7248
7477
|
readFile: async ({ filePath }) => storage.readOfflineSyncFile(filePath),
|
|
7478
|
+
readFileDigest: async ({ filePath }) => {
|
|
7479
|
+
const hash = createHash("sha256");
|
|
7480
|
+
let bytes = 0;
|
|
7481
|
+
for await (const rawChunk of readOfflineSyncFileChunks({
|
|
7482
|
+
filePath,
|
|
7483
|
+
memoryDir,
|
|
7484
|
+
secureStoreKey,
|
|
7485
|
+
chunkSize: OFFLINE_SYNC_FILE_CONTENT_UPLOAD_CHUNK_BYTES
|
|
7486
|
+
})) {
|
|
7487
|
+
const chunk = Buffer.isBuffer(rawChunk) ? rawChunk : Buffer.from(rawChunk);
|
|
7488
|
+
hash.update(chunk);
|
|
7489
|
+
bytes += chunk.length;
|
|
7490
|
+
}
|
|
7491
|
+
return {
|
|
7492
|
+
sha256: hash.digest("hex"),
|
|
7493
|
+
bytes
|
|
7494
|
+
};
|
|
7495
|
+
},
|
|
7249
7496
|
readFileChunks: ({ filePath, chunkSize }) => readOfflineSyncFileChunks({
|
|
7250
7497
|
filePath,
|
|
7251
7498
|
memoryDir,
|
|
@@ -7253,6 +7500,8 @@ async function createOfflineStorageIo(memoryDir) {
|
|
|
7253
7500
|
chunkSize
|
|
7254
7501
|
}),
|
|
7255
7502
|
writeFile: async ({ filePath, content }) => storage.writeOfflineSyncFile(filePath, content),
|
|
7503
|
+
writeStagingFile: async ({ filePath, content }) => storage.writeOfflineSyncStagingFile(filePath, content),
|
|
7504
|
+
writeFileChunks: async ({ filePath, chunks }) => storage.writeOfflineSyncFileChunks(filePath, chunks),
|
|
7256
7505
|
deleteFile: async ({ filePath }) => storage.deleteOfflineSyncFile(filePath)
|
|
7257
7506
|
};
|
|
7258
7507
|
}
|
|
@@ -7366,6 +7615,39 @@ var OfflineLargeFilePushError = class extends Error {
|
|
|
7366
7615
|
this.failures = failures;
|
|
7367
7616
|
}
|
|
7368
7617
|
};
|
|
7618
|
+
function advanceOfflineBaseFilesForSuccessfulPush(options) {
|
|
7619
|
+
const next = offlineFileStateMap(options.baseFiles);
|
|
7620
|
+
const current = offlineFileStateMap(options.currentFiles);
|
|
7621
|
+
const conflictPaths = new Set((options.conflicts ?? []).map((conflict) => conflict.path));
|
|
7622
|
+
for (const relPath of options.directPushedPaths ?? []) {
|
|
7623
|
+
if (conflictPaths.has(relPath)) continue;
|
|
7624
|
+
const file = current.get(relPath);
|
|
7625
|
+
if (file) next.set(relPath, file);
|
|
7626
|
+
}
|
|
7627
|
+
for (const change of options.changeset.changes) {
|
|
7628
|
+
if (conflictPaths.has(change.path)) continue;
|
|
7629
|
+
if (change.type === "delete") {
|
|
7630
|
+
next.delete(change.path);
|
|
7631
|
+
} else {
|
|
7632
|
+
next.set(change.path, {
|
|
7633
|
+
path: change.file.path,
|
|
7634
|
+
sha256: change.file.sha256,
|
|
7635
|
+
bytes: change.file.bytes,
|
|
7636
|
+
mtimeMs: change.file.mtimeMs
|
|
7637
|
+
});
|
|
7638
|
+
}
|
|
7639
|
+
}
|
|
7640
|
+
for (const file of options.hydratedFiles ?? []) {
|
|
7641
|
+
if (conflictPaths.has(file.path)) continue;
|
|
7642
|
+
next.set(file.path, {
|
|
7643
|
+
path: file.path,
|
|
7644
|
+
sha256: file.sha256,
|
|
7645
|
+
bytes: file.bytes,
|
|
7646
|
+
mtimeMs: file.mtimeMs
|
|
7647
|
+
});
|
|
7648
|
+
}
|
|
7649
|
+
return [...next.values()].sort((left, right) => left.path.localeCompare(right.path));
|
|
7650
|
+
}
|
|
7369
7651
|
async function runOfflineSyncOnce(options) {
|
|
7370
7652
|
fs7.mkdirSync(options.memoryDir, { recursive: true });
|
|
7371
7653
|
let activeStatePath = options.statePath;
|
|
@@ -7404,21 +7686,23 @@ async function runOfflineSyncOnce(options) {
|
|
|
7404
7686
|
});
|
|
7405
7687
|
}
|
|
7406
7688
|
const baseFiles = priorState?.baseFiles ?? [];
|
|
7689
|
+
const baseCapturedAt = priorState ? new Date(priorState.lastSyncedAt) : void 0;
|
|
7407
7690
|
const storageIo = await createOfflineStorageIo(options.memoryDir);
|
|
7408
7691
|
const localSourceId = localOfflineSourceId(options.memoryDir);
|
|
7409
|
-
const
|
|
7692
|
+
const currentSnapshotForPush = await buildOfflineSyncSnapshotFromBase({
|
|
7410
7693
|
root: options.memoryDir,
|
|
7411
7694
|
sourceId: localSourceId,
|
|
7412
7695
|
baseFiles,
|
|
7413
|
-
|
|
7414
|
-
readFile: storageIo.readFile
|
|
7415
|
-
});
|
|
7416
|
-
const currentSnapshotForPush = await buildOfflineSyncSnapshot({
|
|
7417
|
-
root: options.memoryDir,
|
|
7418
|
-
sourceId: localSourceId,
|
|
7696
|
+
baseCapturedAt,
|
|
7419
7697
|
includeContent: false,
|
|
7420
7698
|
includeTranscripts: options.includeTranscripts,
|
|
7421
|
-
readFile: storageIo.readFile
|
|
7699
|
+
readFile: storageIo.readFile,
|
|
7700
|
+
readFileDigest: storageIo.readFileDigest
|
|
7701
|
+
});
|
|
7702
|
+
const pendingSummary = summarizeOfflineSyncPendingFiles({
|
|
7703
|
+
baseFiles,
|
|
7704
|
+
currentFiles: currentSnapshotForPush.files,
|
|
7705
|
+
includeTranscripts: options.includeTranscripts
|
|
7422
7706
|
});
|
|
7423
7707
|
const baseByPath = offlineFileStateMap(baseFiles);
|
|
7424
7708
|
let directPushAppliedUpserts = 0;
|
|
@@ -7426,6 +7710,7 @@ async function runOfflineSyncOnce(options) {
|
|
|
7426
7710
|
let directPushNamespace;
|
|
7427
7711
|
const directPushConflicts = [];
|
|
7428
7712
|
const directPushedPaths = /* @__PURE__ */ new Set();
|
|
7713
|
+
const directPushDeferredPaths = /* @__PURE__ */ new Set();
|
|
7429
7714
|
const directPushFailures = [];
|
|
7430
7715
|
for (const file of offlineDirectPushFiles({
|
|
7431
7716
|
currentFiles: currentSnapshotForPush.files,
|
|
@@ -7446,6 +7731,10 @@ async function runOfflineSyncOnce(options) {
|
|
|
7446
7731
|
readFileChunks: storageIo.readFileChunks
|
|
7447
7732
|
});
|
|
7448
7733
|
} catch (error) {
|
|
7734
|
+
if (isOfflineLocalFileChangedError(error)) {
|
|
7735
|
+
directPushDeferredPaths.add(file.path);
|
|
7736
|
+
continue;
|
|
7737
|
+
}
|
|
7449
7738
|
directPushFailures.push({
|
|
7450
7739
|
path: file.path,
|
|
7451
7740
|
error: error instanceof Error ? error.message : String(error)
|
|
@@ -7465,100 +7754,343 @@ async function runOfflineSyncOnce(options) {
|
|
|
7465
7754
|
if (result.skipped) directPushSkipped += 1;
|
|
7466
7755
|
}
|
|
7467
7756
|
}
|
|
7468
|
-
|
|
7469
|
-
|
|
7470
|
-
|
|
7471
|
-
|
|
7472
|
-
root: options.memoryDir,
|
|
7757
|
+
let changeset = {
|
|
7758
|
+
format: "remnic.offline-sync.changeset.v1",
|
|
7759
|
+
schemaVersion: 1,
|
|
7760
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
7473
7761
|
sourceId: localSourceId,
|
|
7474
|
-
baseFiles,
|
|
7475
|
-
excludePaths: [...directPushedPaths],
|
|
7476
7762
|
includeTranscripts: options.includeTranscripts,
|
|
7477
|
-
|
|
7478
|
-
}
|
|
7479
|
-
|
|
7480
|
-
|
|
7481
|
-
|
|
7482
|
-
|
|
7483
|
-
|
|
7484
|
-
|
|
7485
|
-
|
|
7486
|
-
namespace: pushedInline?.namespace ?? directPushNamespace ?? syncNamespace ?? "",
|
|
7487
|
-
appliedUpserts: (pushedInline?.appliedUpserts ?? 0) + directPushAppliedUpserts,
|
|
7488
|
-
appliedDeletes: pushedInline?.appliedDeletes ?? 0,
|
|
7489
|
-
skipped: (pushedInline?.skipped ?? 0) + directPushSkipped,
|
|
7490
|
-
conflicts: [...directPushConflicts, ...pushedInline?.conflicts ?? []]
|
|
7763
|
+
changes: []
|
|
7764
|
+
};
|
|
7765
|
+
let pushed = null;
|
|
7766
|
+
const buildPushedSummary = (pushedInline2) => directPushedPaths.size > 0 || pushedInline2 ? {
|
|
7767
|
+
namespace: pushedInline2?.namespace ?? directPushNamespace ?? syncNamespace ?? "",
|
|
7768
|
+
appliedUpserts: (pushedInline2?.appliedUpserts ?? 0) + directPushAppliedUpserts,
|
|
7769
|
+
appliedDeletes: pushedInline2?.appliedDeletes ?? 0,
|
|
7770
|
+
skipped: (pushedInline2?.skipped ?? 0) + directPushSkipped,
|
|
7771
|
+
conflicts: [...directPushConflicts, ...pushedInline2?.conflicts ?? []]
|
|
7491
7772
|
} : null;
|
|
7492
|
-
const
|
|
7493
|
-
|
|
7494
|
-
|
|
7495
|
-
|
|
7496
|
-
|
|
7497
|
-
|
|
7773
|
+
const mergeInlinePushSummary = (prior, result) => ({
|
|
7774
|
+
namespace: result.namespace || prior?.namespace || syncNamespace || "",
|
|
7775
|
+
appliedUpserts: (prior?.appliedUpserts ?? 0) + result.appliedUpserts,
|
|
7776
|
+
appliedDeletes: (prior?.appliedDeletes ?? 0) + result.appliedDeletes,
|
|
7777
|
+
skipped: (prior?.skipped ?? 0) + result.skipped,
|
|
7778
|
+
conflicts: [...prior?.conflicts ?? [], ...result.conflicts]
|
|
7498
7779
|
});
|
|
7499
|
-
|
|
7500
|
-
|
|
7501
|
-
sourceId: localSourceId,
|
|
7502
|
-
includeContent: false,
|
|
7503
|
-
includeTranscripts: options.includeTranscripts,
|
|
7504
|
-
readFile: storageIo.readFile
|
|
7505
|
-
});
|
|
7506
|
-
const directHydratedPaths = await directHydrateLargeOfflineFiles({
|
|
7507
|
-
remoteUrl: options.remoteUrl,
|
|
7508
|
-
token: options.token,
|
|
7509
|
-
namespace: syncNamespace,
|
|
7510
|
-
includeTranscripts: options.includeTranscripts,
|
|
7511
|
-
snapshot: remoteSnapshotMetadata,
|
|
7512
|
-
baseFiles,
|
|
7513
|
-
currentFiles: currentSnapshot.files,
|
|
7780
|
+
pushed = buildPushedSummary(null);
|
|
7781
|
+
const stateWritePathsFor = (resolvedNamespace2) => offlineStatePathsForNamespace({
|
|
7514
7782
|
memoryDir: options.memoryDir,
|
|
7515
|
-
|
|
7783
|
+
remoteUrl: options.remoteUrl,
|
|
7784
|
+
requestedNamespace: options.namespace,
|
|
7785
|
+
resolvedNamespace: resolvedNamespace2,
|
|
7786
|
+
explicitStatePath: options.statePathExplicit ? activeStatePath : void 0
|
|
7516
7787
|
});
|
|
7517
|
-
const
|
|
7788
|
+
const writePartialPushState = async (error, partial, checkpointChangeset = changeset) => {
|
|
7789
|
+
const resolvedNamespace2 = partial?.resolvedNamespace ?? resolvedOfflineSnapshotNamespace({ namespace: pushed?.namespace ?? "" }, syncNamespace);
|
|
7790
|
+
const stateWritePaths2 = stateWritePathsFor(resolvedNamespace2);
|
|
7791
|
+
const nextBaseFiles = advanceOfflineBaseFilesForSuccessfulPush({
|
|
7792
|
+
baseFiles,
|
|
7793
|
+
currentFiles: currentSnapshotForPush.files,
|
|
7794
|
+
directPushedPaths: [...directPushedPaths],
|
|
7795
|
+
hydratedFiles: partial?.hydratedFiles,
|
|
7796
|
+
changeset: checkpointChangeset,
|
|
7797
|
+
conflicts: pushed?.conflicts ?? directPushConflicts
|
|
7798
|
+
});
|
|
7799
|
+
const state2 = {
|
|
7800
|
+
version: 1,
|
|
7801
|
+
remoteId: options.remoteUrl,
|
|
7802
|
+
...resolvedNamespace2 ? { namespace: resolvedNamespace2 } : {},
|
|
7803
|
+
includeTranscripts: options.includeTranscripts,
|
|
7804
|
+
// Partial checkpoints do not recapture conflicted/deferred paths, so keep
|
|
7805
|
+
// the original capture time for safe fast-base reuse on the next run.
|
|
7806
|
+
lastSyncedAt: priorState?.lastSyncedAt ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
7807
|
+
baseFiles: nextBaseFiles
|
|
7808
|
+
};
|
|
7809
|
+
for (const statePath of stateWritePaths2) {
|
|
7810
|
+
await writeOfflineSyncState(statePath, state2);
|
|
7811
|
+
}
|
|
7812
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
7813
|
+
return {
|
|
7814
|
+
statePath: stateWritePaths2[0] ?? activeStatePath,
|
|
7815
|
+
namespace: resolvedNamespace2,
|
|
7816
|
+
prepared: priorState === null,
|
|
7817
|
+
pushed,
|
|
7818
|
+
pull: null,
|
|
7819
|
+
pullError: message,
|
|
7820
|
+
partial: true,
|
|
7821
|
+
pendingSummary,
|
|
7822
|
+
remoteFileCount: partial?.remoteFileCount ?? null,
|
|
7823
|
+
deferred: {
|
|
7824
|
+
localChangedDuringPush: [...directPushDeferredPaths].sort(),
|
|
7825
|
+
remoteChangedDuringHydrate: [...partial?.remoteDeferredPaths ?? []].sort(),
|
|
7826
|
+
total: directPushDeferredPaths.size + (partial?.remoteDeferredPaths?.length ?? 0)
|
|
7827
|
+
}
|
|
7828
|
+
};
|
|
7829
|
+
};
|
|
7830
|
+
if (directPushFailures.length > 0) {
|
|
7831
|
+
const error = new OfflineLargeFilePushError(directPushFailures);
|
|
7832
|
+
if (pushed) return writePartialPushState(error);
|
|
7833
|
+
throw error;
|
|
7834
|
+
}
|
|
7835
|
+
let currentSnapshotForChangeset = directPushedPaths.size > 0 ? await buildOfflineSyncSnapshotFromBase({
|
|
7518
7836
|
root: options.memoryDir,
|
|
7519
7837
|
sourceId: localSourceId,
|
|
7838
|
+
baseFiles,
|
|
7839
|
+
baseCapturedAt,
|
|
7520
7840
|
includeContent: false,
|
|
7521
7841
|
includeTranscripts: options.includeTranscripts,
|
|
7522
|
-
readFile: storageIo.readFile
|
|
7523
|
-
|
|
7524
|
-
|
|
7525
|
-
|
|
7526
|
-
|
|
7527
|
-
|
|
7528
|
-
|
|
7529
|
-
|
|
7530
|
-
|
|
7531
|
-
|
|
7532
|
-
|
|
7533
|
-
|
|
7534
|
-
|
|
7842
|
+
readFile: storageIo.readFile,
|
|
7843
|
+
readFileDigest: storageIo.readFileDigest
|
|
7844
|
+
}) : currentSnapshotForPush;
|
|
7845
|
+
let changesetRetryCount = 0;
|
|
7846
|
+
for (; ; ) {
|
|
7847
|
+
try {
|
|
7848
|
+
changeset = await buildOfflineSyncChangesetFromSnapshot({
|
|
7849
|
+
root: options.memoryDir,
|
|
7850
|
+
sourceId: localSourceId,
|
|
7851
|
+
currentFiles: currentSnapshotForChangeset.files,
|
|
7852
|
+
baseFiles,
|
|
7853
|
+
excludePaths: [...directPushedPaths, ...directPushDeferredPaths],
|
|
7854
|
+
includeTranscripts: options.includeTranscripts,
|
|
7855
|
+
readFile: storageIo.readFile
|
|
7856
|
+
});
|
|
7857
|
+
break;
|
|
7858
|
+
} catch (error) {
|
|
7859
|
+
const changedPath = offlineChangesetFileChangedPath(error);
|
|
7860
|
+
if (!changedPath) {
|
|
7861
|
+
if (pushed) return writePartialPushState(error);
|
|
7862
|
+
throw error;
|
|
7863
|
+
}
|
|
7864
|
+
if (directPushDeferredPaths.has(changedPath)) {
|
|
7865
|
+
const stalledError = new Error(`offline sync changeset retry stalled on already-deferred path: ${changedPath}`);
|
|
7866
|
+
if (pushed) return writePartialPushState(stalledError);
|
|
7867
|
+
throw stalledError;
|
|
7868
|
+
}
|
|
7869
|
+
if (changesetRetryCount >= OFFLINE_SYNC_CHANGESET_RETRY_MAX) {
|
|
7870
|
+
const retryError = new Error(
|
|
7871
|
+
`offline sync changeset retry limit exceeded after ${OFFLINE_SYNC_CHANGESET_RETRY_MAX} volatile files; last changed path: ${changedPath}`
|
|
7872
|
+
);
|
|
7873
|
+
if (pushed) return writePartialPushState(retryError);
|
|
7874
|
+
throw retryError;
|
|
7875
|
+
}
|
|
7876
|
+
changesetRetryCount += 1;
|
|
7877
|
+
directPushDeferredPaths.add(changedPath);
|
|
7878
|
+
currentSnapshotForChangeset = await buildOfflineSyncSnapshotFromBase({
|
|
7879
|
+
root: options.memoryDir,
|
|
7880
|
+
sourceId: localSourceId,
|
|
7881
|
+
baseFiles,
|
|
7882
|
+
baseCapturedAt,
|
|
7883
|
+
includeContent: false,
|
|
7884
|
+
includeTranscripts: options.includeTranscripts,
|
|
7885
|
+
readFile: storageIo.readFile,
|
|
7886
|
+
readFileDigest: storageIo.readFileDigest
|
|
7887
|
+
});
|
|
7888
|
+
}
|
|
7889
|
+
}
|
|
7890
|
+
const inlineAppliedChanges = [];
|
|
7891
|
+
let pushedInlineProgress = null;
|
|
7892
|
+
let pushedInline = null;
|
|
7535
7893
|
try {
|
|
7536
|
-
|
|
7894
|
+
pushedInline = changeset.changes.length > 0 ? await pushOfflineChanges({
|
|
7895
|
+
remoteUrl: options.remoteUrl,
|
|
7896
|
+
token: options.token,
|
|
7897
|
+
namespace: syncNamespace,
|
|
7898
|
+
changeset,
|
|
7899
|
+
onBatchApplied: (batch) => {
|
|
7900
|
+
inlineAppliedChanges.push(...batch.changeset.changes);
|
|
7901
|
+
pushedInlineProgress = mergeInlinePushSummary(pushedInlineProgress, batch.result);
|
|
7902
|
+
pushed = buildPushedSummary(pushedInlineProgress);
|
|
7903
|
+
}
|
|
7904
|
+
}) : null;
|
|
7905
|
+
} catch (error) {
|
|
7906
|
+
if (pushed || inlineAppliedChanges.length > 0) {
|
|
7907
|
+
return writePartialPushState(error, void 0, {
|
|
7908
|
+
...changeset,
|
|
7909
|
+
changes: inlineAppliedChanges
|
|
7910
|
+
});
|
|
7911
|
+
}
|
|
7912
|
+
throw error;
|
|
7913
|
+
}
|
|
7914
|
+
pushed = buildPushedSummary(pushedInline);
|
|
7915
|
+
let remoteSnapshotMetadata;
|
|
7916
|
+
try {
|
|
7917
|
+
remoteSnapshotMetadata = await fetchOfflineSnapshot({
|
|
7918
|
+
remoteUrl: options.remoteUrl,
|
|
7919
|
+
token: options.token,
|
|
7920
|
+
namespace: syncNamespace,
|
|
7921
|
+
includeTranscripts: options.includeTranscripts,
|
|
7922
|
+
includeContent: false,
|
|
7923
|
+
baseFiles,
|
|
7924
|
+
baseCapturedAt
|
|
7925
|
+
});
|
|
7926
|
+
} catch (error) {
|
|
7927
|
+
if (pushed) return writePartialPushState(error);
|
|
7928
|
+
throw error;
|
|
7929
|
+
}
|
|
7930
|
+
let currentSnapshot;
|
|
7931
|
+
try {
|
|
7932
|
+
currentSnapshot = await buildOfflineSyncSnapshotFromBase({
|
|
7537
7933
|
root: options.memoryDir,
|
|
7538
|
-
|
|
7934
|
+
sourceId: localSourceId,
|
|
7539
7935
|
baseFiles,
|
|
7936
|
+
baseCapturedAt,
|
|
7937
|
+
includeContent: false,
|
|
7938
|
+
includeTranscripts: options.includeTranscripts,
|
|
7540
7939
|
readFile: storageIo.readFile,
|
|
7940
|
+
readFileDigest: storageIo.readFileDigest
|
|
7941
|
+
});
|
|
7942
|
+
} catch (error) {
|
|
7943
|
+
if (pushed) return writePartialPushState(error);
|
|
7944
|
+
throw error;
|
|
7945
|
+
}
|
|
7946
|
+
let directHydration;
|
|
7947
|
+
const directHydrationProgress = {
|
|
7948
|
+
hydratedPaths: /* @__PURE__ */ new Set(),
|
|
7949
|
+
deferredPaths: /* @__PURE__ */ new Set()
|
|
7950
|
+
};
|
|
7951
|
+
try {
|
|
7952
|
+
directHydration = await directHydrateLargeOfflineFiles({
|
|
7953
|
+
remoteUrl: options.remoteUrl,
|
|
7954
|
+
token: options.token,
|
|
7955
|
+
namespace: syncNamespace,
|
|
7956
|
+
includeTranscripts: options.includeTranscripts,
|
|
7957
|
+
snapshot: remoteSnapshotMetadata,
|
|
7958
|
+
baseFiles,
|
|
7959
|
+
currentFiles: currentSnapshot.files,
|
|
7960
|
+
memoryDir: options.memoryDir,
|
|
7961
|
+
readFile: storageIo.readFile,
|
|
7962
|
+
readFileDigest: storageIo.readFileDigest,
|
|
7541
7963
|
writeFile: storageIo.writeFile,
|
|
7542
|
-
|
|
7964
|
+
writeStagingFile: storageIo.writeStagingFile,
|
|
7965
|
+
writeFileChunks: storageIo.writeFileChunks,
|
|
7966
|
+
hydrationProgress: directHydrationProgress
|
|
7543
7967
|
});
|
|
7544
7968
|
} catch (error) {
|
|
7545
|
-
|
|
7546
|
-
|
|
7969
|
+
const partial = offlinePartialHydrationForPaths({
|
|
7970
|
+
files: remoteSnapshotMetadata.files,
|
|
7971
|
+
hydratedPaths: directHydrationProgress.hydratedPaths,
|
|
7972
|
+
deferredPaths: directHydrationProgress.deferredPaths
|
|
7973
|
+
});
|
|
7974
|
+
if (pushed || partial.hydratedFiles.length > 0) {
|
|
7975
|
+
return writePartialPushState(error, {
|
|
7976
|
+
...partial,
|
|
7977
|
+
resolvedNamespace: resolvedOfflineSnapshotNamespace(remoteSnapshotMetadata, syncNamespace),
|
|
7978
|
+
remoteFileCount: remoteSnapshotMetadata.files.length
|
|
7979
|
+
});
|
|
7980
|
+
}
|
|
7981
|
+
throw error;
|
|
7982
|
+
}
|
|
7983
|
+
const directHydratedPaths = directHydration.hydratedPaths;
|
|
7984
|
+
const remoteDeferredPaths = directHydration.deferredPaths;
|
|
7985
|
+
const partialHydration = offlinePartialHydrationForPaths({
|
|
7986
|
+
files: remoteSnapshotMetadata.files,
|
|
7987
|
+
hydratedPaths: directHydratedPaths,
|
|
7988
|
+
deferredPaths: remoteDeferredPaths
|
|
7989
|
+
});
|
|
7990
|
+
const partialHydrationWithContext = {
|
|
7991
|
+
...partialHydration,
|
|
7992
|
+
resolvedNamespace: resolvedOfflineSnapshotNamespace(remoteSnapshotMetadata, syncNamespace),
|
|
7993
|
+
remoteFileCount: remoteSnapshotMetadata.files.length
|
|
7994
|
+
};
|
|
7995
|
+
const buildCurrentSnapshotForApply = async () => buildOfflineSyncSnapshotFromBase({
|
|
7996
|
+
root: options.memoryDir,
|
|
7997
|
+
sourceId: localSourceId,
|
|
7998
|
+
baseFiles,
|
|
7999
|
+
baseCapturedAt,
|
|
8000
|
+
includeContent: false,
|
|
8001
|
+
includeTranscripts: options.includeTranscripts,
|
|
8002
|
+
readFile: storageIo.readFile,
|
|
8003
|
+
readFileDigest: storageIo.readFileDigest
|
|
8004
|
+
});
|
|
8005
|
+
const applyCurrentSnapshot = directHydratedPaths.size > 0 ? await buildCurrentSnapshotForApply() : currentSnapshot;
|
|
8006
|
+
let remoteSnapshot;
|
|
8007
|
+
try {
|
|
8008
|
+
remoteSnapshot = await hydrateOfflineSnapshotContent({
|
|
7547
8009
|
remoteUrl: options.remoteUrl,
|
|
7548
8010
|
token: options.token,
|
|
7549
8011
|
namespace: syncNamespace,
|
|
7550
8012
|
includeTranscripts: options.includeTranscripts,
|
|
7551
8013
|
snapshot: remoteSnapshotMetadata,
|
|
7552
|
-
baseFiles
|
|
8014
|
+
baseFiles,
|
|
8015
|
+
currentFiles: applyCurrentSnapshot.files,
|
|
8016
|
+
deferredPaths: [...remoteDeferredPaths]
|
|
7553
8017
|
});
|
|
8018
|
+
} catch (error) {
|
|
8019
|
+
if (pushed || partialHydration.hydratedFiles.length > 0) {
|
|
8020
|
+
return writePartialPushState(error, partialHydrationWithContext);
|
|
8021
|
+
}
|
|
8022
|
+
throw error;
|
|
8023
|
+
}
|
|
8024
|
+
const resolvedNamespace = resolvedOfflineSnapshotNamespace(remoteSnapshot, syncNamespace);
|
|
8025
|
+
let pull;
|
|
8026
|
+
try {
|
|
8027
|
+
const latestApplySnapshot = await buildCurrentSnapshotForApply();
|
|
7554
8028
|
pull = await applyOfflineSyncSnapshot({
|
|
7555
8029
|
root: options.memoryDir,
|
|
7556
|
-
snapshot:
|
|
8030
|
+
snapshot: remoteSnapshot,
|
|
7557
8031
|
baseFiles,
|
|
8032
|
+
currentFiles: latestApplySnapshot.files,
|
|
8033
|
+
deferredPaths: [...remoteDeferredPaths],
|
|
8034
|
+
allowMissingConflictContent: true,
|
|
7558
8035
|
readFile: storageIo.readFile,
|
|
8036
|
+
readFileDigest: storageIo.readFileDigest,
|
|
7559
8037
|
writeFile: storageIo.writeFile,
|
|
7560
8038
|
deleteFile: storageIo.deleteFile
|
|
7561
8039
|
});
|
|
8040
|
+
} catch (error) {
|
|
8041
|
+
if (!isMissingOfflineContentError(error)) {
|
|
8042
|
+
if (pushed || partialHydration.hydratedFiles.length > 0) {
|
|
8043
|
+
return writePartialPushState(error, {
|
|
8044
|
+
...partialHydrationWithContext,
|
|
8045
|
+
resolvedNamespace
|
|
8046
|
+
});
|
|
8047
|
+
}
|
|
8048
|
+
throw error;
|
|
8049
|
+
}
|
|
8050
|
+
let retrySnapshot;
|
|
8051
|
+
try {
|
|
8052
|
+
retrySnapshot = await hydrateOfflineSnapshotContent({
|
|
8053
|
+
remoteUrl: options.remoteUrl,
|
|
8054
|
+
token: options.token,
|
|
8055
|
+
namespace: syncNamespace,
|
|
8056
|
+
includeTranscripts: options.includeTranscripts,
|
|
8057
|
+
snapshot: remoteSnapshotMetadata,
|
|
8058
|
+
baseFiles,
|
|
8059
|
+
currentFiles: applyCurrentSnapshot.files,
|
|
8060
|
+
deferredPaths: [...remoteDeferredPaths]
|
|
8061
|
+
});
|
|
8062
|
+
} catch (retryError) {
|
|
8063
|
+
if (pushed || partialHydration.hydratedFiles.length > 0) {
|
|
8064
|
+
return writePartialPushState(retryError, {
|
|
8065
|
+
...partialHydrationWithContext,
|
|
8066
|
+
resolvedNamespace
|
|
8067
|
+
});
|
|
8068
|
+
}
|
|
8069
|
+
throw retryError;
|
|
8070
|
+
}
|
|
8071
|
+
try {
|
|
8072
|
+
const latestRetryApplySnapshot = await buildCurrentSnapshotForApply();
|
|
8073
|
+
pull = await applyOfflineSyncSnapshot({
|
|
8074
|
+
root: options.memoryDir,
|
|
8075
|
+
snapshot: retrySnapshot,
|
|
8076
|
+
baseFiles,
|
|
8077
|
+
currentFiles: latestRetryApplySnapshot.files,
|
|
8078
|
+
deferredPaths: [...remoteDeferredPaths],
|
|
8079
|
+
allowMissingConflictContent: true,
|
|
8080
|
+
readFile: storageIo.readFile,
|
|
8081
|
+
readFileDigest: storageIo.readFileDigest,
|
|
8082
|
+
writeFile: storageIo.writeFile,
|
|
8083
|
+
deleteFile: storageIo.deleteFile
|
|
8084
|
+
});
|
|
8085
|
+
} catch (retryApplyError) {
|
|
8086
|
+
if (pushed || partialHydration.hydratedFiles.length > 0) {
|
|
8087
|
+
return writePartialPushState(retryApplyError, {
|
|
8088
|
+
...partialHydrationWithContext,
|
|
8089
|
+
resolvedNamespace
|
|
8090
|
+
});
|
|
8091
|
+
}
|
|
8092
|
+
throw retryApplyError;
|
|
8093
|
+
}
|
|
7562
8094
|
}
|
|
7563
8095
|
const state = offlineSyncStateFromSnapshot({
|
|
7564
8096
|
remoteId: options.remoteUrl,
|
|
@@ -7566,13 +8098,7 @@ async function runOfflineSyncOnce(options) {
|
|
|
7566
8098
|
snapshot: remoteSnapshot,
|
|
7567
8099
|
baseFiles: pull.nextBaseFiles
|
|
7568
8100
|
});
|
|
7569
|
-
const stateWritePaths =
|
|
7570
|
-
memoryDir: options.memoryDir,
|
|
7571
|
-
remoteUrl: options.remoteUrl,
|
|
7572
|
-
requestedNamespace: options.namespace,
|
|
7573
|
-
resolvedNamespace,
|
|
7574
|
-
explicitStatePath: options.statePathExplicit ? activeStatePath : void 0
|
|
7575
|
-
});
|
|
8101
|
+
const stateWritePaths = stateWritePathsFor(resolvedNamespace);
|
|
7576
8102
|
for (const statePath of stateWritePaths) {
|
|
7577
8103
|
await writeOfflineSyncState(statePath, state);
|
|
7578
8104
|
}
|
|
@@ -7582,8 +8108,53 @@ async function runOfflineSyncOnce(options) {
|
|
|
7582
8108
|
prepared: priorState === null,
|
|
7583
8109
|
pushed,
|
|
7584
8110
|
pull,
|
|
8111
|
+
partial: false,
|
|
7585
8112
|
pendingSummary,
|
|
7586
|
-
remoteFileCount: remoteSnapshot.files.length
|
|
8113
|
+
remoteFileCount: remoteSnapshot.files.length,
|
|
8114
|
+
deferred: {
|
|
8115
|
+
localChangedDuringPush: [...directPushDeferredPaths].sort(),
|
|
8116
|
+
remoteChangedDuringHydrate: [...remoteDeferredPaths].sort(),
|
|
8117
|
+
total: directPushDeferredPaths.size + remoteDeferredPaths.size
|
|
8118
|
+
}
|
|
8119
|
+
};
|
|
8120
|
+
}
|
|
8121
|
+
function sumOfflineFileBytes(files) {
|
|
8122
|
+
return files.reduce((total, file) => total + file.bytes, 0);
|
|
8123
|
+
}
|
|
8124
|
+
function offlineStateJsonSummary(state) {
|
|
8125
|
+
if (!state) return null;
|
|
8126
|
+
return {
|
|
8127
|
+
remoteId: state.remoteId,
|
|
8128
|
+
namespace: state.namespace ?? null,
|
|
8129
|
+
includeTranscripts: state.includeTranscripts,
|
|
8130
|
+
lastSyncedAt: state.lastSyncedAt,
|
|
8131
|
+
baseFileCount: state.baseFiles.length,
|
|
8132
|
+
baseBytes: sumOfflineFileBytes(state.baseFiles)
|
|
8133
|
+
};
|
|
8134
|
+
}
|
|
8135
|
+
function offlinePullJsonSummary(pull) {
|
|
8136
|
+
return {
|
|
8137
|
+
upserted: pull.upserted,
|
|
8138
|
+
deleted: pull.deleted,
|
|
8139
|
+
skipped: pull.skipped,
|
|
8140
|
+
pendingLocal: pull.pendingLocal,
|
|
8141
|
+
conflicts: pull.conflicts,
|
|
8142
|
+
nextBaseFileCount: pull.nextBaseFiles.length,
|
|
8143
|
+
nextBaseBytes: sumOfflineFileBytes(pull.nextBaseFiles)
|
|
8144
|
+
};
|
|
8145
|
+
}
|
|
8146
|
+
function offlineSyncResultJsonSummary(result) {
|
|
8147
|
+
return {
|
|
8148
|
+
statePath: result.statePath,
|
|
8149
|
+
namespace: result.namespace ?? null,
|
|
8150
|
+
prepared: result.prepared,
|
|
8151
|
+
partial: result.partial,
|
|
8152
|
+
pushed: result.pushed,
|
|
8153
|
+
pull: result.pull ? offlinePullJsonSummary(result.pull) : null,
|
|
8154
|
+
pullError: result.pullError ?? null,
|
|
8155
|
+
pendingSummary: result.pendingSummary,
|
|
8156
|
+
remoteFileCount: result.remoteFileCount,
|
|
8157
|
+
deferred: result.deferred
|
|
7587
8158
|
};
|
|
7588
8159
|
}
|
|
7589
8160
|
function assertOfflineStateMatches(options) {
|
|
@@ -7664,6 +8235,7 @@ Environment fallbacks:
|
|
|
7664
8235
|
snapshot: remoteSnapshot,
|
|
7665
8236
|
baseFiles: existingState?.state.baseFiles ?? [],
|
|
7666
8237
|
readFile: storageIo.readFile,
|
|
8238
|
+
readFileDigest: storageIo.readFileDigest,
|
|
7667
8239
|
writeFile: storageIo.writeFile,
|
|
7668
8240
|
deleteFile: storageIo.deleteFile
|
|
7669
8241
|
});
|
|
@@ -7677,7 +8249,12 @@ Environment fallbacks:
|
|
|
7677
8249
|
await writeOfflineSyncState(pathToWrite, state);
|
|
7678
8250
|
}
|
|
7679
8251
|
if (json) {
|
|
7680
|
-
console.log(JSON.stringify({
|
|
8252
|
+
console.log(JSON.stringify({
|
|
8253
|
+
statePath: activeStatePath,
|
|
8254
|
+
namespace: resolvedNamespace,
|
|
8255
|
+
remoteFiles: remoteSnapshot.files.length,
|
|
8256
|
+
pull: offlinePullJsonSummary(pull)
|
|
8257
|
+
}, null, 2));
|
|
7681
8258
|
} else {
|
|
7682
8259
|
console.log(`Offline cache prepared: ${memoryDir}`);
|
|
7683
8260
|
console.log(`Namespace: ${resolvedNamespace ?? "(default)"}`);
|
|
@@ -7699,12 +8276,19 @@ Environment fallbacks:
|
|
|
7699
8276
|
statePathExplicit
|
|
7700
8277
|
});
|
|
7701
8278
|
if (json) {
|
|
7702
|
-
console.log(JSON.stringify(result, null, 2));
|
|
8279
|
+
console.log(JSON.stringify(offlineSyncResultJsonSummary(result), null, 2));
|
|
7703
8280
|
} else {
|
|
7704
8281
|
console.log(`Offline sync complete${result.prepared ? " (initialized state)" : ""}.`);
|
|
7705
8282
|
console.log(`Pushed: ${result.pushed ? `${result.pushed.appliedUpserts} upserts, ${result.pushed.appliedDeletes} deletes, ${result.pushed.conflicts.length} conflicts` : "nothing pending"}`);
|
|
7706
|
-
|
|
8283
|
+
if (result.pull) {
|
|
8284
|
+
console.log(`Pulled: ${result.pull.upserted} upserts, ${result.pull.deleted} deletes, ${result.pull.conflicts.length} conflicts`);
|
|
8285
|
+
} else {
|
|
8286
|
+
console.log(`Pulled: deferred (${result.pullError ?? "pull unavailable"})`);
|
|
8287
|
+
}
|
|
7707
8288
|
console.log(`Pending local before push: ${result.pendingSummary.total}`);
|
|
8289
|
+
if (result.deferred.total > 0) {
|
|
8290
|
+
console.log(`Deferred volatile files: ${result.deferred.total}`);
|
|
8291
|
+
}
|
|
7708
8292
|
console.log(`Namespace: ${result.namespace ?? "(default)"}`);
|
|
7709
8293
|
console.log(`State: ${result.statePath}`);
|
|
7710
8294
|
}
|
|
@@ -7727,11 +8311,17 @@ Environment fallbacks:
|
|
|
7727
8311
|
root: memoryDir,
|
|
7728
8312
|
sourceId: localOfflineSourceId(memoryDir),
|
|
7729
8313
|
baseFiles: state?.baseFiles ?? [],
|
|
8314
|
+
baseCapturedAt: state ? new Date(state.lastSyncedAt) : void 0,
|
|
7730
8315
|
includeTranscripts,
|
|
7731
|
-
readFile: storageIo.readFile
|
|
8316
|
+
readFile: storageIo.readFile,
|
|
8317
|
+
readFileDigest: storageIo.readFileDigest
|
|
7732
8318
|
});
|
|
7733
8319
|
if (json) {
|
|
7734
|
-
console.log(JSON.stringify({
|
|
8320
|
+
console.log(JSON.stringify({
|
|
8321
|
+
statePath: statePath ?? null,
|
|
8322
|
+
state: offlineStateJsonSummary(state),
|
|
8323
|
+
pending: summary
|
|
8324
|
+
}, null, 2));
|
|
7735
8325
|
} else {
|
|
7736
8326
|
console.log(`Offline state: ${state ? "ready" : "not prepared"}`);
|
|
7737
8327
|
console.log(`State: ${statePath ?? "(not selected; pass --state or --remote-url to inspect a prepared remote state)"}`);
|
|
@@ -7762,8 +8352,10 @@ Environment fallbacks:
|
|
|
7762
8352
|
statePath,
|
|
7763
8353
|
statePathExplicit
|
|
7764
8354
|
});
|
|
8355
|
+
const pulled = result.pull ? result.pull.upserted + result.pull.deleted : 0;
|
|
8356
|
+
const conflicts = (result.pushed?.conflicts.length ?? 0) + (result.pull?.conflicts.length ?? 0);
|
|
7765
8357
|
console.log(
|
|
7766
|
-
`[${(/* @__PURE__ */ new Date()).toISOString()}] sync ok: pushed=${result.pushed ? result.pushed.appliedUpserts + result.pushed.appliedDeletes : 0}, pulled=${
|
|
8358
|
+
`[${(/* @__PURE__ */ new Date()).toISOString()}] sync ${result.partial ? "partial" : "ok"}: pushed=${result.pushed ? result.pushed.appliedUpserts + result.pushed.appliedDeletes : 0}, pulled=${pulled}, conflicts=${conflicts}, deferred=${result.deferred.total}${result.pullError ? `, pullError=${result.pullError}` : ""}`
|
|
7767
8359
|
);
|
|
7768
8360
|
} catch (error) {
|
|
7769
8361
|
console.log(`[${(/* @__PURE__ */ new Date()).toISOString()}] sync waiting: ${error instanceof Error ? error.message : String(error)}`);
|
|
@@ -10676,32 +11268,46 @@ if (argv1Base.endsWith("remnic.ts") || argv1Base.endsWith("remnic.js") || argv1B
|
|
|
10676
11268
|
export {
|
|
10677
11269
|
BENCHMARK_CATALOG,
|
|
10678
11270
|
OFFLINE_SYNC_APPLY_MAX_REQUEST_BYTES,
|
|
11271
|
+
OFFLINE_SYNC_CHANGESET_RETRY_MAX,
|
|
10679
11272
|
OFFLINE_SYNC_DIRECT_HYDRATE_MIN_BYTES,
|
|
10680
11273
|
OFFLINE_SYNC_DIRECT_PUSH_MIN_BYTES,
|
|
10681
11274
|
OFFLINE_SYNC_FILE_CONTENT_UPLOAD_CHUNK_BYTES,
|
|
11275
|
+
OFFLINE_SYNC_REQUEST_TIMEOUT_DEFAULT_MS,
|
|
10682
11276
|
TAXONOMY_RESOLVE_BOOLEAN_FLAGS,
|
|
10683
11277
|
TAXONOMY_RESOLVE_VALUE_FLAGS,
|
|
10684
11278
|
__benchDatasetTestHooks,
|
|
11279
|
+
advanceOfflineBaseFilesForSuccessfulPush,
|
|
10685
11280
|
buildBenchRuntimeProfileRequest,
|
|
10686
11281
|
buildPackageBenchExecutionPlans,
|
|
10687
11282
|
buildQueryRecallRequest,
|
|
10688
11283
|
chunkOfflineChangesetApplyBatches,
|
|
10689
11284
|
chunkOfflineFileContentBatches,
|
|
11285
|
+
directHydrateLargeOfflineFiles,
|
|
10690
11286
|
extractXrayRawArgs,
|
|
10691
11287
|
formatOfflineLargeFilePushFailureMessage,
|
|
11288
|
+
formatOfflineRequestForError,
|
|
10692
11289
|
getBenchUsageText,
|
|
10693
11290
|
hasFlag,
|
|
11291
|
+
isOfflineSnapshotPostFallbackError,
|
|
10694
11292
|
main,
|
|
11293
|
+
offlinePartialHydrationForPaths,
|
|
11294
|
+
offlineSnapshotBasePostBody,
|
|
11295
|
+
offlineSnapshotBasePostBodyFits,
|
|
11296
|
+
offlineSnapshotContentFilesForApply,
|
|
10695
11297
|
parseBenchArgs,
|
|
10696
11298
|
parseCapsuleForkArgs,
|
|
10697
11299
|
parseConnectorConfig,
|
|
11300
|
+
parseOfflineSyncRequestTimeoutMs,
|
|
10698
11301
|
parseTaxonomyResolveArgs,
|
|
10699
11302
|
parseTrainingExportArgs,
|
|
11303
|
+
pushOfflineFileContent,
|
|
11304
|
+
pushOfflineFileContentFromChunkReader,
|
|
10700
11305
|
renderQueryTextLines,
|
|
10701
11306
|
resolveConfigPath,
|
|
10702
11307
|
resolveFlag,
|
|
10703
11308
|
resolveMemoryDir,
|
|
10704
11309
|
resolveSyncSourceDir,
|
|
11310
|
+
runOfflineSyncOnce,
|
|
10705
11311
|
runTrainingExport,
|
|
10706
11312
|
runXrayCommand,
|
|
10707
11313
|
shouldDirectHydrateOfflineFile,
|