@ucdjs/ucd-store 0.0.1 → 1.0.1-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -0
- package/dist/context-DfX5D4Fb.mjs +201 -0
- package/dist/index.d.mts +747 -0
- package/dist/index.mjs +1356 -0
- package/package.json +36 -13
- package/dist/index.d.ts +0 -4
- package/dist/index.js +0 -7
package/dist/index.mjs
ADDED
|
@@ -0,0 +1,1356 @@
|
|
|
1
|
+
import { a as isUCDStoreInternalContext, c as UCDStoreBridgeUnsupportedOperation, d as UCDStoreGenericError, f as UCDStoreVersionNotFoundError, i as extractFilterPatterns, l as UCDStoreFileNotFoundError, n as createInternalContext, o as UCDStoreApiFallbackError, r as createPublicContext, s as UCDStoreBaseError, u as UCDStoreFilterError } from "./context-DfX5D4Fb.mjs";
|
|
2
|
+
import { UCDJS_API_BASE_URL, UCDJS_STORE_BASE_URL } from "@ucdjs/env";
|
|
3
|
+
import { patheBasename, patheDirname, patheJoin, patheResolve } from "@ucdjs/path-utils";
|
|
4
|
+
import { createConcurrencyLimiter, createDebugger, createPathFilter, discoverEndpointsFromConfig, filterTreeStructure, flattenFilePaths, getDefaultUCDEndpointConfig, normalizeTreeForFiltering, tryOr, wrapTry } from "@ucdjs-internal/shared";
|
|
5
|
+
import { createUCDClientWithConfig } from "@ucdjs/client";
|
|
6
|
+
import { computeFileHash, computeFileHashWithoutUCDHeader, getLockfilePath, readLockfile, readLockfileOrUndefined, readSnapshotOrUndefined, writeLockfile, writeSnapshot } from "@ucdjs/lockfile";
|
|
7
|
+
import defu from "defu";
|
|
8
|
+
import { hasUCDFolderPath } from "@unicode-utils/core";
|
|
9
|
+
import { prependLeadingSlash } from "@luxass/utils";
|
|
10
|
+
import { assertCapability, hasCapability } from "@ucdjs/fs-bridge";
|
|
11
|
+
|
|
12
|
+
//#region src/files/get.ts
|
|
13
|
+
const debug$10 = createDebugger("ucdjs:ucd-store:files:get");
|
|
14
|
+
/**
|
|
15
|
+
* Retrieves a specific file for a Unicode version from the local store.
|
|
16
|
+
* By default, only reads files that are actually present in the store.
|
|
17
|
+
* Optionally caches the file to local FS after fetching from API (if allowApi is enabled).
|
|
18
|
+
*
|
|
19
|
+
* @this {InternalUCDStoreContext} - Internal store context with client, filters, FS bridge, and configuration
|
|
20
|
+
* @param {string} version - The Unicode version containing the file
|
|
21
|
+
* @param {string} filePath - The path to the file within the version
|
|
22
|
+
* @param {GetFileOptions} [options] - Optional filters, cache behavior, and API fallback
|
|
23
|
+
* @returns {Promise<OperationResult<string, StoreError>>} Operation result with file content or error
|
|
24
|
+
*/
|
|
25
|
+
async function _getFile(version, filePath, options) {
|
|
26
|
+
return wrapTry(async () => {
|
|
27
|
+
if (!this.versions.resolved.includes(version)) throw new UCDStoreVersionNotFoundError(version);
|
|
28
|
+
if (!this.filter(filePath, options?.filters)) {
|
|
29
|
+
debug$10?.("File '%s' does not pass filters", filePath);
|
|
30
|
+
throw new UCDStoreFilterError(`File '${filePath}' does not pass filters`, {
|
|
31
|
+
excludePattern: options?.filters?.exclude || [],
|
|
32
|
+
includePattern: options?.filters?.include || [],
|
|
33
|
+
filePath
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
const localPath = patheJoin(version, filePath);
|
|
37
|
+
debug$10?.("Checking local file existence:", localPath);
|
|
38
|
+
if (await this.fs.exists(localPath)) {
|
|
39
|
+
debug$10?.("Local file exists:", localPath);
|
|
40
|
+
const content = await tryOr({
|
|
41
|
+
try: this.fs.read(localPath),
|
|
42
|
+
err: (err) => {
|
|
43
|
+
debug$10?.("Failed to read local file:", localPath, err);
|
|
44
|
+
if (!options?.allowApi) throw new UCDStoreGenericError(`Failed to read file '${filePath}' from local store`, {
|
|
45
|
+
version,
|
|
46
|
+
filePath
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
});
|
|
50
|
+
if (content != null) {
|
|
51
|
+
debug$10?.("Returning local file content:", localPath);
|
|
52
|
+
return content;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
if (!options?.allowApi) throw new UCDStoreGenericError(`File '${filePath}' does not exist in local store`, {
|
|
56
|
+
version,
|
|
57
|
+
filePath
|
|
58
|
+
});
|
|
59
|
+
debug$10?.("Fetching file from API:", filePath);
|
|
60
|
+
const remotePath = patheJoin(version, hasUCDFolderPath(version) ? "ucd" : "", filePath);
|
|
61
|
+
const result = await this.client.files.get(remotePath);
|
|
62
|
+
if (result.error) throw new UCDStoreApiFallbackError({
|
|
63
|
+
version,
|
|
64
|
+
filePath,
|
|
65
|
+
status: result.error.status,
|
|
66
|
+
reason: "fetch-failed",
|
|
67
|
+
message: `Failed to fetch file '${filePath}': ${result.error.message}`
|
|
68
|
+
});
|
|
69
|
+
if (result.data == null) throw new UCDStoreApiFallbackError({
|
|
70
|
+
version,
|
|
71
|
+
filePath,
|
|
72
|
+
reason: "no-data"
|
|
73
|
+
});
|
|
74
|
+
let content;
|
|
75
|
+
if (typeof result.data === "string") content = result.data;
|
|
76
|
+
else content = JSON.stringify(result.data);
|
|
77
|
+
return content;
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
function getFile(thisOrContext, versionOrPath, pathOrOptions, options) {
|
|
81
|
+
if (isUCDStoreInternalContext(thisOrContext)) return _getFile.call(thisOrContext, versionOrPath, pathOrOptions, options);
|
|
82
|
+
return _getFile.call(this, thisOrContext, versionOrPath, pathOrOptions);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
//#endregion
|
|
86
|
+
//#region src/files/list.ts
|
|
87
|
+
const debug$9 = createDebugger("ucdjs:ucd-store:files:list");
|
|
88
|
+
/**
|
|
89
|
+
* Build a lookup of normalized paths to original paths.
|
|
90
|
+
* This allows us to filter using normalized paths but return original paths.
|
|
91
|
+
*/
|
|
92
|
+
function buildPathMapping(version, originalEntries, normalizedEntries) {
|
|
93
|
+
const originalPaths = flattenFilePaths(originalEntries);
|
|
94
|
+
const normalizedPaths = flattenFilePaths(normalizedEntries);
|
|
95
|
+
const mapping = /* @__PURE__ */ new Map();
|
|
96
|
+
for (let i = 0; i < normalizedPaths.length; i++) {
|
|
97
|
+
const normalizedPath = normalizedPaths[i];
|
|
98
|
+
const originalPath = originalPaths[i];
|
|
99
|
+
if (normalizedPath && originalPath) mapping.set(normalizedPath, originalPath);
|
|
100
|
+
}
|
|
101
|
+
return mapping;
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* Lists all file paths for a Unicode version. The operation prefers the
|
|
105
|
+
* configured file system bridge and can optionally fall back to the API when
|
|
106
|
+
* the bridge path is missing or cannot be read.
|
|
107
|
+
*
|
|
108
|
+
* Returns full paths (e.g., "/16.0.0/UnicodeData.txt"), not just filenames.
|
|
109
|
+
*
|
|
110
|
+
* @this {InternalUCDStoreContext} - Internal store context
|
|
111
|
+
* @param version Unicode version to list files for
|
|
112
|
+
* @param options Optional filters and `allowApi` fallback behavior
|
|
113
|
+
* @returns Operation result containing full file paths or an error
|
|
114
|
+
*/
|
|
115
|
+
async function _listFiles(version, options) {
|
|
116
|
+
return wrapTry(async () => {
|
|
117
|
+
if (!this.versions.resolved.includes(version)) throw new UCDStoreVersionNotFoundError(version);
|
|
118
|
+
const filesPath = version;
|
|
119
|
+
debug$9?.("Using files path:", filesPath, "for version:", version);
|
|
120
|
+
if (await this.fs.exists(filesPath)) try {
|
|
121
|
+
const entries = await this.fs.listdir(filesPath, true);
|
|
122
|
+
debug$9?.("Listed entries from store for version:", version);
|
|
123
|
+
const normalizedEntries = normalizeTreeForFiltering(version, entries);
|
|
124
|
+
const pathMapping = buildPathMapping(version, entries, normalizedEntries);
|
|
125
|
+
const originalPaths = flattenFilePaths(filterTreeStructure(this.filter, normalizedEntries, options?.filters)).map((normalizedPath) => {
|
|
126
|
+
const original = pathMapping.get(normalizedPath);
|
|
127
|
+
if (!original) return `/${version}/${normalizedPath.replace(/^\/+/, "")}`;
|
|
128
|
+
return original;
|
|
129
|
+
});
|
|
130
|
+
debug$9?.("Listed %d files from store for version: %s", originalPaths.length, version);
|
|
131
|
+
return originalPaths;
|
|
132
|
+
} catch (err) {
|
|
133
|
+
debug$9?.("Failed to list directory:", filesPath, err);
|
|
134
|
+
if (!options?.allowApi) return [];
|
|
135
|
+
}
|
|
136
|
+
if (!options?.allowApi) {
|
|
137
|
+
debug$9?.("Directory does not exist and allowApi is false:", filesPath);
|
|
138
|
+
return [];
|
|
139
|
+
}
|
|
140
|
+
debug$9?.("Fetching file tree from API for version:", version);
|
|
141
|
+
const result = await this.client.versions.getFileTree(version);
|
|
142
|
+
if (result.error) throw new UCDStoreApiFallbackError({
|
|
143
|
+
version,
|
|
144
|
+
filePath: "file-tree",
|
|
145
|
+
reason: "fetch-failed",
|
|
146
|
+
status: result.error.status
|
|
147
|
+
});
|
|
148
|
+
if (result.data == null) throw new UCDStoreApiFallbackError({
|
|
149
|
+
version,
|
|
150
|
+
filePath: "file-tree",
|
|
151
|
+
reason: "no-data"
|
|
152
|
+
});
|
|
153
|
+
const normalizedTree = normalizeTreeForFiltering(version, result.data);
|
|
154
|
+
const pathMapping = buildPathMapping(version, result.data, normalizedTree);
|
|
155
|
+
return flattenFilePaths(filterTreeStructure(this.filter, normalizedTree, options?.filters)).map((normalizedPath) => {
|
|
156
|
+
const original = pathMapping.get(normalizedPath);
|
|
157
|
+
if (!original) return `/${version}/${normalizedPath.replace(/^\/+/, "")}`;
|
|
158
|
+
return original;
|
|
159
|
+
});
|
|
160
|
+
});
|
|
161
|
+
}
|
|
162
|
+
function listFiles(thisOrContext, versionOrOptions, options) {
|
|
163
|
+
if (isUCDStoreInternalContext(thisOrContext)) return _listFiles.call(thisOrContext, versionOrOptions, options);
|
|
164
|
+
return _listFiles.call(this, thisOrContext, versionOrOptions);
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
//#endregion
|
|
168
|
+
//#region src/files/tree.ts
|
|
169
|
+
const debug$8 = createDebugger("ucdjs:ucd-store:files:tree");
|
|
170
|
+
/**
|
|
171
|
+
* Converts FSEntry array to UnicodeFileTreeNode array by adding lastModified field.
|
|
172
|
+
*/
|
|
173
|
+
function fsEntryToFileTreeNode(entries) {
|
|
174
|
+
return entries.map((entry) => {
|
|
175
|
+
if (entry.type === "directory") return {
|
|
176
|
+
type: "directory",
|
|
177
|
+
name: entry.name,
|
|
178
|
+
path: entry.path,
|
|
179
|
+
lastModified: null,
|
|
180
|
+
children: fsEntryToFileTreeNode(entry.children)
|
|
181
|
+
};
|
|
182
|
+
return {
|
|
183
|
+
type: "file",
|
|
184
|
+
name: entry.name,
|
|
185
|
+
path: entry.path,
|
|
186
|
+
lastModified: null
|
|
187
|
+
};
|
|
188
|
+
});
|
|
189
|
+
}
|
|
190
|
+
/**
|
|
191
|
+
* Retrieves the file tree for a specific Unicode version from the local store.
|
|
192
|
+
* By default, only returns the tree structure for files actually present in the store.
|
|
193
|
+
* Applies global filters and optional method-specific filters to the tree.
|
|
194
|
+
*
|
|
195
|
+
* @this {InternalUCDStoreContext} - Internal store context with client, filters, FS bridge, and configuration
|
|
196
|
+
* @param {string} version - The Unicode version to fetch the file tree for
|
|
197
|
+
* @param {GetFileTreeOptions} [options] - Optional filters and API fallback behavior
|
|
198
|
+
* @returns {Promise<OperationResult<UnicodeFileTreeNode[], StoreError>>} Operation result with filtered file tree or error
|
|
199
|
+
*/
|
|
200
|
+
async function _getFileTree(version, options) {
|
|
201
|
+
return wrapTry(async () => {
|
|
202
|
+
if (!this.versions.resolved.includes(version)) throw new UCDStoreVersionNotFoundError(version);
|
|
203
|
+
const localPath = version;
|
|
204
|
+
if (await this.fs.exists(localPath)) try {
|
|
205
|
+
const normalizedEntries = normalizeTreeForFiltering(version, fsEntryToFileTreeNode(await this.fs.listdir(localPath, true)));
|
|
206
|
+
return filterTreeStructure(this.filter, normalizedEntries, {
|
|
207
|
+
exclude: options?.filters?.exclude,
|
|
208
|
+
include: options?.filters?.include
|
|
209
|
+
});
|
|
210
|
+
} catch (err) {
|
|
211
|
+
debug$8?.("Failed to list local directory:", localPath, err);
|
|
212
|
+
if (!options?.allowApi) return [];
|
|
213
|
+
}
|
|
214
|
+
if (!options?.allowApi) {
|
|
215
|
+
debug$8?.("Directory does not exist locally and allowApi is false:", localPath);
|
|
216
|
+
return [];
|
|
217
|
+
}
|
|
218
|
+
debug$8?.("Fetching file tree from API for version:", version);
|
|
219
|
+
const result = await this.client.versions.getFileTree(version);
|
|
220
|
+
if (result.error) throw new UCDStoreApiFallbackError({
|
|
221
|
+
version,
|
|
222
|
+
filePath: "file-tree",
|
|
223
|
+
reason: "fetch-failed",
|
|
224
|
+
status: result.error.status
|
|
225
|
+
});
|
|
226
|
+
if (result.data == null) throw new UCDStoreApiFallbackError({
|
|
227
|
+
version,
|
|
228
|
+
filePath: "file-tree",
|
|
229
|
+
reason: "no-data"
|
|
230
|
+
});
|
|
231
|
+
const normalizedTree = normalizeTreeForFiltering(version, result.data);
|
|
232
|
+
return filterTreeStructure(this.filter, normalizedTree, {
|
|
233
|
+
exclude: options?.filters?.exclude,
|
|
234
|
+
include: options?.filters?.include
|
|
235
|
+
});
|
|
236
|
+
});
|
|
237
|
+
}
|
|
238
|
+
function getFileTree(thisOrContext, versionOrOptions, options) {
|
|
239
|
+
if (isUCDStoreInternalContext(thisOrContext)) return _getFileTree.call(thisOrContext, versionOrOptions, options);
|
|
240
|
+
return _getFileTree.call(this, thisOrContext, versionOrOptions);
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
//#endregion
|
|
244
|
+
//#region src/utils/reports.ts
|
|
245
|
+
function createEmptyFileCounts() {
|
|
246
|
+
return {
|
|
247
|
+
total: 0,
|
|
248
|
+
success: 0,
|
|
249
|
+
skipped: 0,
|
|
250
|
+
failed: 0
|
|
251
|
+
};
|
|
252
|
+
}
|
|
253
|
+
function createEmptyOperationMetrics() {
|
|
254
|
+
return {
|
|
255
|
+
successRate: 0,
|
|
256
|
+
cacheHitRate: 0,
|
|
257
|
+
failureRate: 0,
|
|
258
|
+
averageTimePerFile: 0
|
|
259
|
+
};
|
|
260
|
+
}
|
|
261
|
+
function createEmptyStorageMetrics() {
|
|
262
|
+
return {
|
|
263
|
+
totalSize: "0 B",
|
|
264
|
+
averageFileSize: "0 B"
|
|
265
|
+
};
|
|
266
|
+
}
|
|
267
|
+
function createEmptySummary(duration = 0) {
|
|
268
|
+
return {
|
|
269
|
+
duration,
|
|
270
|
+
counts: createEmptyFileCounts(),
|
|
271
|
+
metrics: createEmptyOperationMetrics(),
|
|
272
|
+
storage: createEmptyStorageMetrics()
|
|
273
|
+
};
|
|
274
|
+
}
|
|
275
|
+
function computeMetrics(counts, duration) {
|
|
276
|
+
const total = counts.total;
|
|
277
|
+
return {
|
|
278
|
+
successRate: total > 0 ? counts.success / total * 100 : 0,
|
|
279
|
+
cacheHitRate: total > 0 ? counts.skipped / total * 100 : 0,
|
|
280
|
+
failureRate: total > 0 ? counts.failed / total * 100 : 0,
|
|
281
|
+
averageTimePerFile: total > 0 ? duration / total : 0
|
|
282
|
+
};
|
|
283
|
+
}
|
|
284
|
+
function formatBytes(bytes) {
|
|
285
|
+
if (bytes === 0) return "0 B";
|
|
286
|
+
const sizes = [
|
|
287
|
+
"B",
|
|
288
|
+
"KB",
|
|
289
|
+
"MB",
|
|
290
|
+
"GB",
|
|
291
|
+
"TB"
|
|
292
|
+
];
|
|
293
|
+
const i = Math.floor(Math.log(bytes) / Math.log(1024));
|
|
294
|
+
const clampedI = Math.max(0, Math.min(i, sizes.length - 1));
|
|
295
|
+
return `${(bytes / 1024 ** clampedI).toFixed(2)} ${sizes[clampedI]}`;
|
|
296
|
+
}
|
|
297
|
+
function computeStorageMetrics(totalBytes, successCount) {
|
|
298
|
+
return {
|
|
299
|
+
totalSize: formatBytes(totalBytes),
|
|
300
|
+
averageFileSize: successCount > 0 ? formatBytes(totalBytes / successCount) : "0 B"
|
|
301
|
+
};
|
|
302
|
+
}
|
|
303
|
+
function aggregateFileCounts(versionReports) {
|
|
304
|
+
const counts = createEmptyFileCounts();
|
|
305
|
+
for (const report of versionReports.values()) {
|
|
306
|
+
counts.total += report.counts.total;
|
|
307
|
+
counts.success += report.counts.success;
|
|
308
|
+
counts.skipped += report.counts.skipped;
|
|
309
|
+
counts.failed += report.counts.failed;
|
|
310
|
+
}
|
|
311
|
+
return counts;
|
|
312
|
+
}
|
|
313
|
+
function createSummaryFromVersionReports(versionReports, duration, totalBytes) {
|
|
314
|
+
const counts = aggregateFileCounts(versionReports);
|
|
315
|
+
return {
|
|
316
|
+
duration,
|
|
317
|
+
counts,
|
|
318
|
+
metrics: computeMetrics(counts, duration),
|
|
319
|
+
storage: computeStorageMetrics(totalBytes, counts.success)
|
|
320
|
+
};
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
//#endregion
|
|
324
|
+
//#region src/reports/analyze.ts
|
|
325
|
+
const debug$7 = createDebugger("ucdjs:ucd-store:analyze");
|
|
326
|
+
/**
|
|
327
|
+
* Analyzes Unicode data in the store.
|
|
328
|
+
*
|
|
329
|
+
* @this {InternalUCDStoreContext} - Internal store context with client, filters, FS bridge, and configuration
|
|
330
|
+
* @param {AnalyzeOptions} [options] - Analyze options
|
|
331
|
+
* @returns {Promise<OperationResult<AnalysisReport, StoreError>>} Operation result
|
|
332
|
+
*/
|
|
333
|
+
async function _analyze(options) {
|
|
334
|
+
return wrapTry(async () => {
|
|
335
|
+
const startTime = Date.now();
|
|
336
|
+
const versionReports = /* @__PURE__ */ new Map();
|
|
337
|
+
const versionsToAnalyze = options?.versions ?? this.versions.resolved;
|
|
338
|
+
for (const version of versionsToAnalyze) {
|
|
339
|
+
if (!this.versions.resolved.includes(version)) continue;
|
|
340
|
+
const versionStartTime = Date.now();
|
|
341
|
+
const errors = [];
|
|
342
|
+
let expectedFiles = [];
|
|
343
|
+
try {
|
|
344
|
+
expectedFiles = await this.getExpectedFilePaths(version);
|
|
345
|
+
debug$7?.("Found expected files while analyzing: %O", expectedFiles.map((f) => f.storePath));
|
|
346
|
+
} catch (err) {
|
|
347
|
+
errors.push({
|
|
348
|
+
name: "expected-files",
|
|
349
|
+
filePath: prependLeadingSlash(version),
|
|
350
|
+
reason: err instanceof Error ? err.message : String(err)
|
|
351
|
+
});
|
|
352
|
+
}
|
|
353
|
+
let [actualFilePaths, listFilesError] = await listFiles(this, version, {
|
|
354
|
+
allowApi: false,
|
|
355
|
+
filters: options?.filters
|
|
356
|
+
});
|
|
357
|
+
if (listFilesError != null) {
|
|
358
|
+
errors.push({
|
|
359
|
+
name: "list-files",
|
|
360
|
+
filePath: prependLeadingSlash(version),
|
|
361
|
+
reason: listFilesError.message
|
|
362
|
+
});
|
|
363
|
+
actualFilePaths = [];
|
|
364
|
+
}
|
|
365
|
+
actualFilePaths = (actualFilePaths || []).filter((filePath) => !filePath.endsWith("snapshot.json"));
|
|
366
|
+
debug$7?.("Actual files while analyzing: %O", actualFilePaths);
|
|
367
|
+
const expectedStorePathsSet = new Set(expectedFiles.map((f) => f.storePath));
|
|
368
|
+
const actualFilePathsSet = new Set(actualFilePaths);
|
|
369
|
+
const expectedFilesByStorePath = new Map(expectedFiles.map((f) => [f.storePath, f]));
|
|
370
|
+
const presentFiles = [];
|
|
371
|
+
const orphanedFiles = [];
|
|
372
|
+
const missingFiles = [];
|
|
373
|
+
const fileTypes = {};
|
|
374
|
+
debug$7?.("Started analyzing files");
|
|
375
|
+
for (const actualFile of actualFilePathsSet) {
|
|
376
|
+
const ext = getExtension(actualFile);
|
|
377
|
+
fileTypes[ext] ??= 0;
|
|
378
|
+
fileTypes[ext] += 1;
|
|
379
|
+
if (expectedStorePathsSet.has(actualFile)) {
|
|
380
|
+
const expectedFile = expectedFilesByStorePath.get(actualFile);
|
|
381
|
+
presentFiles.push({
|
|
382
|
+
name: expectedFile?.name || actualFile.split("/").pop() || actualFile,
|
|
383
|
+
filePath: actualFile
|
|
384
|
+
});
|
|
385
|
+
} else orphanedFiles.push({
|
|
386
|
+
name: actualFile.split("/").pop() || actualFile,
|
|
387
|
+
filePath: actualFile
|
|
388
|
+
});
|
|
389
|
+
}
|
|
390
|
+
for (const expectedFile of expectedFiles) if (!actualFilePathsSet.has(expectedFile.storePath)) missingFiles.push({
|
|
391
|
+
name: expectedFile.name,
|
|
392
|
+
filePath: expectedFile.storePath
|
|
393
|
+
});
|
|
394
|
+
debug$7?.("Finished analyzing files");
|
|
395
|
+
presentFiles.sort((a, b) => a.name.localeCompare(b.name));
|
|
396
|
+
orphanedFiles.sort((a, b) => a.name.localeCompare(b.name));
|
|
397
|
+
missingFiles.sort((a, b) => a.name.localeCompare(b.name));
|
|
398
|
+
const isComplete = orphanedFiles.length === 0 && missingFiles.length === 0 && errors.length === 0;
|
|
399
|
+
const counts = {
|
|
400
|
+
total: expectedFiles.length,
|
|
401
|
+
success: presentFiles.length,
|
|
402
|
+
skipped: orphanedFiles.length,
|
|
403
|
+
failed: missingFiles.length
|
|
404
|
+
};
|
|
405
|
+
const metrics = computeMetrics(counts, Date.now() - versionStartTime);
|
|
406
|
+
const versionReport = {
|
|
407
|
+
version,
|
|
408
|
+
isComplete,
|
|
409
|
+
files: {
|
|
410
|
+
present: presentFiles,
|
|
411
|
+
orphaned: orphanedFiles,
|
|
412
|
+
missing: missingFiles
|
|
413
|
+
},
|
|
414
|
+
counts,
|
|
415
|
+
metrics,
|
|
416
|
+
errors,
|
|
417
|
+
fileTypes
|
|
418
|
+
};
|
|
419
|
+
versionReports.set(version, versionReport);
|
|
420
|
+
}
|
|
421
|
+
const summary = createSummaryFromVersionReports(versionReports, Date.now() - startTime, 0);
|
|
422
|
+
return {
|
|
423
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
424
|
+
versions: versionReports,
|
|
425
|
+
summary
|
|
426
|
+
};
|
|
427
|
+
});
|
|
428
|
+
}
|
|
429
|
+
function analyze(thisOrContext, options) {
|
|
430
|
+
if (isUCDStoreInternalContext(thisOrContext)) return _analyze.call(thisOrContext, options);
|
|
431
|
+
return _analyze.call(this, thisOrContext);
|
|
432
|
+
}
|
|
433
|
+
function getExtension(filePath) {
|
|
434
|
+
const match = filePath.match(/\.[^/.]+$/);
|
|
435
|
+
return match ? match[0] : "no_extension";
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
//#endregion
|
|
439
|
+
//#region src/reports/compare.ts
|
|
440
|
+
const debug$6 = createDebugger("ucdjs:ucd-store:compare");
|
|
441
|
+
/**
|
|
442
|
+
* Resolves comparison mode into separate modes for from/to versions.
|
|
443
|
+
*/
|
|
444
|
+
function resolveModes(mode) {
|
|
445
|
+
if (!mode) return {
|
|
446
|
+
fromMode: "prefer-local",
|
|
447
|
+
toMode: "prefer-local"
|
|
448
|
+
};
|
|
449
|
+
if (Array.isArray(mode)) return {
|
|
450
|
+
fromMode: mode[0],
|
|
451
|
+
toMode: mode[1]
|
|
452
|
+
};
|
|
453
|
+
return {
|
|
454
|
+
fromMode: mode,
|
|
455
|
+
toMode: mode
|
|
456
|
+
};
|
|
457
|
+
}
|
|
458
|
+
/**
|
|
459
|
+
* Converts a mode to the allowApi option for listFiles/getFile.
|
|
460
|
+
* - "prefer-local": allowApi = true (try local first, fall back to API)
|
|
461
|
+
* - "local": allowApi = false (only local)
|
|
462
|
+
* - "api": allowApi = true (API-only, handled by forcing fetch from API)
|
|
463
|
+
*/
|
|
464
|
+
function modeToAllowApi(mode) {
|
|
465
|
+
return mode !== "local";
|
|
466
|
+
}
|
|
467
|
+
/**
|
|
468
|
+
* Normalizes a file path by removing the version prefix.
|
|
469
|
+
* Handles paths like "/16.0.0/UnicodeData.txt" or "16.0.0/UnicodeData.txt"
|
|
470
|
+
*/
|
|
471
|
+
function normalizeFilePath(filePath, version) {
|
|
472
|
+
const versionPrefix = `/${version}/`;
|
|
473
|
+
const versionPrefixNoSlash = `${version}/`;
|
|
474
|
+
if (filePath.startsWith(versionPrefix)) return filePath.slice(versionPrefix.length);
|
|
475
|
+
if (filePath.startsWith(versionPrefixNoSlash)) return filePath.slice(versionPrefixNoSlash.length);
|
|
476
|
+
return filePath.replace(/^\/+/, "");
|
|
477
|
+
}
|
|
478
|
+
/**
|
|
479
|
+
* Compares two versions in the store and returns a report describing added/removed/modified/unchanged files.
|
|
480
|
+
*
|
|
481
|
+
* The comparison uses content hashes (SHA-256 with Unicode header stripped) to determine
|
|
482
|
+
* if files have actually changed content, not just metadata like version numbers in headers.
|
|
483
|
+
*
|
|
484
|
+
* @this {InternalUCDStoreContext} - Internal store context with client, filters, FS bridge, and configuration
|
|
485
|
+
* @param {CompareOptions} options - Compare options with from/to versions
|
|
486
|
+
* @returns {Promise<OperationResult<VersionComparison, StoreError>>} Operation result with comparison report
|
|
487
|
+
*/
|
|
488
|
+
async function _compare(options) {
|
|
489
|
+
return wrapTry(async () => {
|
|
490
|
+
if (!options) throw new UCDStoreGenericError("Options with `from` and `to` versions are required");
|
|
491
|
+
const { from, to } = options;
|
|
492
|
+
if (!from || !to) throw new UCDStoreGenericError("Both `from` and `to` versions must be specified");
|
|
493
|
+
if (!this.versions.resolved.includes(from)) throw new UCDStoreVersionNotFoundError(from);
|
|
494
|
+
if (!this.versions.resolved.includes(to)) throw new UCDStoreVersionNotFoundError(to);
|
|
495
|
+
const { fromMode, toMode } = resolveModes(options.mode);
|
|
496
|
+
const includeHashes = options.includeFileHashes !== false;
|
|
497
|
+
const concurrency = options.concurrency ?? 5;
|
|
498
|
+
debug$6?.("Comparing %s -> %s (modes: %s, %s, includeHashes: %s)", from, to, fromMode, toMode, includeHashes);
|
|
499
|
+
const [fromFilesRaw, fromError] = await listFiles(this, from, {
|
|
500
|
+
allowApi: modeToAllowApi(fromMode),
|
|
501
|
+
filters: options.filters
|
|
502
|
+
});
|
|
503
|
+
if (fromError) throw fromError;
|
|
504
|
+
const [toFilesRaw, toError] = await listFiles(this, to, {
|
|
505
|
+
allowApi: modeToAllowApi(toMode),
|
|
506
|
+
filters: options.filters
|
|
507
|
+
});
|
|
508
|
+
if (toError) throw toError;
|
|
509
|
+
const fromFiles = (fromFilesRaw || []).map((p) => normalizeFilePath(p, from)).filter((p) => p !== "snapshot.json");
|
|
510
|
+
const toFiles = (toFilesRaw || []).map((p) => normalizeFilePath(p, to)).filter((p) => p !== "snapshot.json");
|
|
511
|
+
debug$6?.("From version %s has %d files, to version %s has %d files", from, fromFiles.length, to, toFiles.length);
|
|
512
|
+
const fromSet = new Set(fromFiles);
|
|
513
|
+
const toSet = new Set(toFiles);
|
|
514
|
+
const added = toFiles.filter((p) => !fromSet.has(p)).sort();
|
|
515
|
+
const removed = fromFiles.filter((p) => !toSet.has(p)).sort();
|
|
516
|
+
const common = fromFiles.filter((p) => toSet.has(p)).sort();
|
|
517
|
+
debug$6?.("Added: %d, Removed: %d, Common: %d", added.length, removed.length, common.length);
|
|
518
|
+
const modified = [];
|
|
519
|
+
const changes = [];
|
|
520
|
+
let unchangedCount = 0;
|
|
521
|
+
if (includeHashes && common.length > 0) {
|
|
522
|
+
const limit = createConcurrencyLimiter(concurrency);
|
|
523
|
+
await Promise.all(common.map((file) => limit(async () => {
|
|
524
|
+
const [fromContent, fromFileError] = await getFile(this, from, file, { allowApi: modeToAllowApi(fromMode) });
|
|
525
|
+
if (fromFileError) {
|
|
526
|
+
debug$6?.("Failed to get file %s from version %s: %s", file, from, fromFileError.message);
|
|
527
|
+
throw fromFileError;
|
|
528
|
+
}
|
|
529
|
+
const [toContent, toFileError] = await getFile(this, to, file, { allowApi: modeToAllowApi(toMode) });
|
|
530
|
+
if (toFileError) {
|
|
531
|
+
debug$6?.("Failed to get file %s from version %s: %s", file, to, toFileError.message);
|
|
532
|
+
throw toFileError;
|
|
533
|
+
}
|
|
534
|
+
const fromHash = await computeFileHashWithoutUCDHeader(fromContent);
|
|
535
|
+
const toHash = await computeFileHashWithoutUCDHeader(toContent);
|
|
536
|
+
const fromSize = new TextEncoder().encode(fromContent).length;
|
|
537
|
+
const toSize = new TextEncoder().encode(toContent).length;
|
|
538
|
+
if (fromHash !== toHash) {
|
|
539
|
+
modified.push(file);
|
|
540
|
+
changes.push({
|
|
541
|
+
file,
|
|
542
|
+
changeType: "content-changed",
|
|
543
|
+
from: {
|
|
544
|
+
size: fromSize,
|
|
545
|
+
hash: fromHash
|
|
546
|
+
},
|
|
547
|
+
to: {
|
|
548
|
+
size: toSize,
|
|
549
|
+
hash: toHash
|
|
550
|
+
}
|
|
551
|
+
});
|
|
552
|
+
debug$6?.("File %s modified: %s -> %s (sizes: %d -> %d)", file, fromHash.slice(0, 16), toHash.slice(0, 16), fromSize, toSize);
|
|
553
|
+
} else unchangedCount++;
|
|
554
|
+
})));
|
|
555
|
+
} else unchangedCount = common.length;
|
|
556
|
+
const result = {
|
|
557
|
+
from,
|
|
558
|
+
to,
|
|
559
|
+
added: Object.freeze(added),
|
|
560
|
+
removed: Object.freeze(removed),
|
|
561
|
+
modified: Object.freeze(modified.sort()),
|
|
562
|
+
unchanged: unchangedCount,
|
|
563
|
+
changes: Object.freeze(changes.sort((a, b) => a.file.localeCompare(b.file)))
|
|
564
|
+
};
|
|
565
|
+
debug$6?.("Comparison complete: %d added, %d removed, %d modified, %d unchanged", result.added.length, result.removed.length, result.modified.length, result.unchanged);
|
|
566
|
+
return result;
|
|
567
|
+
});
|
|
568
|
+
}
|
|
569
|
+
function compare(thisOrContext, options) {
|
|
570
|
+
if (isUCDStoreInternalContext(thisOrContext)) return _compare.call(thisOrContext, options);
|
|
571
|
+
return _compare.call(this, thisOrContext);
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
//#endregion
|
|
575
|
+
//#region src/tasks/mirror.ts
|
|
576
|
+
const debug$5 = createDebugger("ucdjs:ucd-store:mirror");
|
|
577
|
+
/**
|
|
578
|
+
* Helper to create a ReportFile from file information.
|
|
579
|
+
*/
|
|
580
|
+
function createReportFile(name, filePath) {
|
|
581
|
+
return {
|
|
582
|
+
name,
|
|
583
|
+
filePath: `/${filePath}`
|
|
584
|
+
};
|
|
585
|
+
}
|
|
586
|
+
/**
|
|
587
|
+
* Helper to create a version report with empty initial state.
|
|
588
|
+
*/
|
|
589
|
+
function createVersionReport(version) {
|
|
590
|
+
return {
|
|
591
|
+
version,
|
|
592
|
+
files: {
|
|
593
|
+
downloaded: [],
|
|
594
|
+
skipped: [],
|
|
595
|
+
failed: []
|
|
596
|
+
},
|
|
597
|
+
counts: {
|
|
598
|
+
total: 0,
|
|
599
|
+
success: 0,
|
|
600
|
+
skipped: 0,
|
|
601
|
+
failed: 0
|
|
602
|
+
},
|
|
603
|
+
metrics: {
|
|
604
|
+
successRate: 0,
|
|
605
|
+
cacheHitRate: 0,
|
|
606
|
+
failureRate: 0,
|
|
607
|
+
averageTimePerFile: 0
|
|
608
|
+
},
|
|
609
|
+
errors: []
|
|
610
|
+
};
|
|
611
|
+
}
|
|
612
|
+
/**
|
|
613
|
+
* Updates a version report's counts and metrics based on its file lists.
|
|
614
|
+
*/
|
|
615
|
+
function finalizeVersionReport(report, duration) {
|
|
616
|
+
report.counts = {
|
|
617
|
+
total: report.files.downloaded.length + report.files.skipped.length + report.files.failed.length,
|
|
618
|
+
success: report.files.downloaded.length,
|
|
619
|
+
skipped: report.files.skipped.length,
|
|
620
|
+
failed: report.files.failed.length
|
|
621
|
+
};
|
|
622
|
+
report.metrics = computeMetrics(report.counts, duration);
|
|
623
|
+
}
|
|
624
|
+
/**
|
|
625
|
+
* Mirrors Unicode data files from the API to local storage.
|
|
626
|
+
* Downloads actual Unicode data files for specified versions.
|
|
627
|
+
*
|
|
628
|
+
* @this {InternalUCDStoreContext} - Internal store context with client, filters, FS bridge, and configuration
|
|
629
|
+
* @param {MirrorOptions} [options] - Mirror options
|
|
630
|
+
* @returns {Promise<OperationResult<MirrorReport, StoreError>>} Operation result
|
|
631
|
+
*/
|
|
632
|
+
async function _mirror(options) {
|
|
633
|
+
return wrapTry(async () => {
|
|
634
|
+
if (!hasCapability(this.fs, ["mkdir", "write"])) throw new UCDStoreGenericError("Filesystem does not support required write operations for mirroring.");
|
|
635
|
+
debug$5?.("Starting mirror operation with context: %O", this);
|
|
636
|
+
const versions = options?.versions ?? this.versions.resolved;
|
|
637
|
+
const concurrency = options?.concurrency ?? 5;
|
|
638
|
+
const force = options?.force ?? false;
|
|
639
|
+
if (versions.length === 0) {
|
|
640
|
+
debug$5?.("No versions to mirror");
|
|
641
|
+
return {
|
|
642
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
643
|
+
versions: /* @__PURE__ */ new Map(),
|
|
644
|
+
summary: createEmptySummary(0)
|
|
645
|
+
};
|
|
646
|
+
}
|
|
647
|
+
const startTime = Date.now();
|
|
648
|
+
const limit = createConcurrencyLimiter(concurrency);
|
|
649
|
+
debug$5?.(`Starting mirror for ${versions.length} version(s) with concurrency=${concurrency}`);
|
|
650
|
+
const versionedReports = /* @__PURE__ */ new Map();
|
|
651
|
+
for (const version of versions) versionedReports.set(version, createVersionReport(version));
|
|
652
|
+
const directoriesToCreate = /* @__PURE__ */ new Set();
|
|
653
|
+
const filesQueue = [];
|
|
654
|
+
for (const version of versions) {
|
|
655
|
+
debug$5?.(`Fetching file tree for version ${version}`);
|
|
656
|
+
const result = await this.client.versions.getFileTree(version);
|
|
657
|
+
debug$5?.(`Fetched file tree for version ${version}`);
|
|
658
|
+
if (result.error) throw new UCDStoreGenericError(`Failed to fetch file tree for version '${version}': ${result.error.message}`, {
|
|
659
|
+
version,
|
|
660
|
+
status: result.error.status
|
|
661
|
+
});
|
|
662
|
+
debug$5?.(`Processing file tree for version ${version}: %O`, result.data);
|
|
663
|
+
if (result.data == null) throw new UCDStoreGenericError(`Failed to fetch file tree for version '${version}': no data returned`, { version });
|
|
664
|
+
const normalizedTree = normalizeTreeForFiltering(version, result.data);
|
|
665
|
+
const filePaths = flattenFilePaths(filterTreeStructure(this.filter, normalizedTree, options?.filters));
|
|
666
|
+
debug$5?.(`Found ${filePaths.length} files for version ${version} after filtering`);
|
|
667
|
+
if (!versionedReports.has(version)) throw new UCDStoreGenericError(`Internal error: missing version report for ${version}`);
|
|
668
|
+
const versionResult = versionedReports.get(version);
|
|
669
|
+
for (const filePath of filePaths) {
|
|
670
|
+
const normalized = filePath.replace(/^\/+/, "");
|
|
671
|
+
const localPath = patheJoin(version, normalized);
|
|
672
|
+
const remotePath = patheJoin(version, hasUCDFolderPath(version) ? "ucd" : "", normalized);
|
|
673
|
+
directoriesToCreate.add(patheDirname(localPath));
|
|
674
|
+
debug$5?.(`Queueing file for mirroring: version=${version}, filePath=${filePath}, normalized=${normalized}, localPath=${localPath}, remotePath=${remotePath}`);
|
|
675
|
+
filesQueue.push({
|
|
676
|
+
name: normalized,
|
|
677
|
+
version,
|
|
678
|
+
filePath: normalized,
|
|
679
|
+
localPath,
|
|
680
|
+
remotePath,
|
|
681
|
+
versionResult
|
|
682
|
+
});
|
|
683
|
+
}
|
|
684
|
+
}
|
|
685
|
+
const totalFiles = filesQueue.length;
|
|
686
|
+
debug$5?.(`Total files to mirror: ${totalFiles}, directories to create: ${directoriesToCreate.size}`);
|
|
687
|
+
let totalDownloadedSize = 0;
|
|
688
|
+
await Promise.all([...directoriesToCreate].map(async (dir) => {
|
|
689
|
+
if (!await this.fs.exists(dir)) await this.fs.mkdir(dir);
|
|
690
|
+
}));
|
|
691
|
+
await Promise.all(filesQueue.map((item) => limit(async () => {
|
|
692
|
+
try {
|
|
693
|
+
if (!force && await this.fs.exists(item.localPath)) {
|
|
694
|
+
item.versionResult.files.skipped.push(createReportFile(item.name, item.localPath));
|
|
695
|
+
return;
|
|
696
|
+
}
|
|
697
|
+
debug$5?.(`Fetching file ${item.remotePath} from API`);
|
|
698
|
+
const { data, error, response } = await this.client.files.get(item.remotePath);
|
|
699
|
+
if (error) throw new UCDStoreGenericError(`Failed to fetch file '${item.filePath}': ${error.message}`);
|
|
700
|
+
if (!data) throw new UCDStoreGenericError(`Failed to fetch file '${item.filePath}': no data returned`);
|
|
701
|
+
const contentSize = +(response?.headers.get("content-length") ?? "0");
|
|
702
|
+
const contentType = response?.headers.get("content-type");
|
|
703
|
+
let content;
|
|
704
|
+
if (typeof data === "string") content = data;
|
|
705
|
+
else if (typeof data === "object" && !Array.isArray(data) && Object.prototype.toString.call(data) === "[object Object]" && contentType === "application/json") content = JSON.stringify(data);
|
|
706
|
+
else throw new UCDStoreGenericError(`Failed to mirror file '${item.filePath}': unsupported data type received`);
|
|
707
|
+
await this.fs.write(item.localPath, content);
|
|
708
|
+
item.versionResult.files.downloaded.push(createReportFile(item.name, item.localPath));
|
|
709
|
+
totalDownloadedSize += contentSize;
|
|
710
|
+
} catch (err) {
|
|
711
|
+
item.versionResult.files.failed.push(createReportFile(item.name, item.localPath));
|
|
712
|
+
item.versionResult.errors.push({
|
|
713
|
+
name: item.name,
|
|
714
|
+
filePath: prependLeadingSlash(item.localPath),
|
|
715
|
+
reason: err instanceof Error ? err.message : String(err)
|
|
716
|
+
});
|
|
717
|
+
debug$5?.(`Failed to mirror file ${item.remotePath} for version ${item.version}:`, err);
|
|
718
|
+
}
|
|
719
|
+
})));
|
|
720
|
+
const duration = Date.now() - startTime;
|
|
721
|
+
for (const report of versionedReports.values()) {
|
|
722
|
+
finalizeVersionReport(report, duration / versions.length);
|
|
723
|
+
report.files.downloaded.sort((a, b) => a.name.localeCompare(b.name));
|
|
724
|
+
report.files.skipped.sort((a, b) => a.name.localeCompare(b.name));
|
|
725
|
+
report.files.failed.sort((a, b) => a.name.localeCompare(b.name));
|
|
726
|
+
}
|
|
727
|
+
const lockfile = await readLockfileOrUndefined(this.fs, this.lockfile.path);
|
|
728
|
+
const updatedLockfileVersions = lockfile ? { ...lockfile.versions } : {};
|
|
729
|
+
const now = /* @__PURE__ */ new Date();
|
|
730
|
+
for (const [version, report] of versionedReports.entries()) {
|
|
731
|
+
const allFiles = [...report.files.downloaded, ...report.files.skipped];
|
|
732
|
+
debug$5?.(`Preparing snapshot for version ${version}, total files: ${allFiles.length}`);
|
|
733
|
+
debug$5?.(`Files: %O`, allFiles);
|
|
734
|
+
if (allFiles.length > 0) {
|
|
735
|
+
debug$5?.(`Creating snapshot for version ${version}`);
|
|
736
|
+
const snapshotFiles = {};
|
|
737
|
+
let totalSize = 0;
|
|
738
|
+
for (const localFile of allFiles) {
|
|
739
|
+
const normalizedPath = localFile.filePath.startsWith(`${version}/`) ? localFile.filePath.slice(version.length + 1) : localFile.filePath;
|
|
740
|
+
debug$5?.(`Processing file for snapshot: version=${version}, localPath=${localFile.filePath}, normalizedPath=${normalizedPath}`);
|
|
741
|
+
const fileContent = await this.fs.read(localFile.filePath);
|
|
742
|
+
if (fileContent) {
|
|
743
|
+
const hash = await computeFileHashWithoutUCDHeader(fileContent);
|
|
744
|
+
const fileHash = await computeFileHash(fileContent);
|
|
745
|
+
const size = new TextEncoder().encode(fileContent).length;
|
|
746
|
+
snapshotFiles[normalizedPath] = {
|
|
747
|
+
hash,
|
|
748
|
+
fileHash,
|
|
749
|
+
size
|
|
750
|
+
};
|
|
751
|
+
totalSize += size;
|
|
752
|
+
}
|
|
753
|
+
}
|
|
754
|
+
await writeSnapshot(this.fs, version, {
|
|
755
|
+
unicodeVersion: version,
|
|
756
|
+
files: snapshotFiles
|
|
757
|
+
});
|
|
758
|
+
const existingEntry = updatedLockfileVersions[version];
|
|
759
|
+
updatedLockfileVersions[version] = {
|
|
760
|
+
path: `${version}/snapshot.json`,
|
|
761
|
+
fileCount: allFiles.length,
|
|
762
|
+
totalSize,
|
|
763
|
+
createdAt: existingEntry?.createdAt ?? now,
|
|
764
|
+
updatedAt: now
|
|
765
|
+
};
|
|
766
|
+
}
|
|
767
|
+
}
|
|
768
|
+
if (Object.keys(updatedLockfileVersions).length > 0) {
|
|
769
|
+
const filters = extractFilterPatterns(this.filter) ?? lockfile?.filters;
|
|
770
|
+
await writeLockfile(this.fs, this.lockfile.path, {
|
|
771
|
+
lockfileVersion: 1,
|
|
772
|
+
createdAt: lockfile?.createdAt ?? now,
|
|
773
|
+
updatedAt: now,
|
|
774
|
+
versions: updatedLockfileVersions,
|
|
775
|
+
filters
|
|
776
|
+
});
|
|
777
|
+
}
|
|
778
|
+
const aggregatedCounts = {
|
|
779
|
+
total: 0,
|
|
780
|
+
success: 0,
|
|
781
|
+
skipped: 0,
|
|
782
|
+
failed: 0
|
|
783
|
+
};
|
|
784
|
+
for (const report of versionedReports.values()) {
|
|
785
|
+
aggregatedCounts.total += report.counts.total;
|
|
786
|
+
aggregatedCounts.success += report.counts.success;
|
|
787
|
+
aggregatedCounts.skipped += report.counts.skipped;
|
|
788
|
+
aggregatedCounts.failed += report.counts.failed;
|
|
789
|
+
}
|
|
790
|
+
const summary = {
|
|
791
|
+
duration,
|
|
792
|
+
counts: aggregatedCounts,
|
|
793
|
+
metrics: computeMetrics(aggregatedCounts, duration),
|
|
794
|
+
storage: computeStorageMetrics(totalDownloadedSize, aggregatedCounts.success)
|
|
795
|
+
};
|
|
796
|
+
debug$5?.(`Mirror completed: %d downloaded, %d skipped, %d failed in ${duration}ms`, summary.counts.success, summary.counts.skipped, summary.counts.failed);
|
|
797
|
+
return {
|
|
798
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
799
|
+
versions: versionedReports,
|
|
800
|
+
summary
|
|
801
|
+
};
|
|
802
|
+
});
|
|
803
|
+
}
|
|
804
|
+
function mirror(thisOrContext, options) {
|
|
805
|
+
if (isUCDStoreInternalContext(thisOrContext)) return _mirror.call(thisOrContext, options);
|
|
806
|
+
return _mirror.call(this, thisOrContext);
|
|
807
|
+
}
|
|
808
|
+
|
|
809
|
+
//#endregion
|
|
810
|
+
//#region src/tasks/sync.ts
|
|
811
|
+
const debug$4 = createDebugger("ucdjs:ucd-store:sync");
|
|
812
|
+
/**
|
|
813
|
+
* Synchronizes the store lockfile with available versions from API and mirrors files.
|
|
814
|
+
* Updates lockfile, downloads missing files, and optionally removes orphaned files/unavailable versions.
|
|
815
|
+
*
|
|
816
|
+
* @this {InternalUCDStoreContext} - Internal store context
|
|
817
|
+
* @param {SyncOptions} [options] - Sync options
|
|
818
|
+
* @returns {Promise<OperationResult<SyncResult, StoreError>>} Operation result
|
|
819
|
+
*/
|
|
820
|
+
async function _sync(options) {
|
|
821
|
+
return wrapTry(async () => {
|
|
822
|
+
const startTime = Date.now();
|
|
823
|
+
if (!hasCapability(this.fs, ["mkdir", "write"])) throw new UCDStoreGenericError("Filesystem does not support required write operations for syncing.");
|
|
824
|
+
const concurrency = options?.concurrency ?? 5;
|
|
825
|
+
const force = options?.force ?? false;
|
|
826
|
+
const removeUnavailable = options?.removeUnavailable ?? false;
|
|
827
|
+
const cleanOrphaned = options?.cleanOrphaned ?? false;
|
|
828
|
+
debug$4?.("Fetching available versions from API");
|
|
829
|
+
const availableVersionsFromApi = await getVersionsFromApi(this);
|
|
830
|
+
debug$4?.(`Found ${availableVersionsFromApi.length} available versions from API`);
|
|
831
|
+
const lockfile = await readLockfileOrUndefined(this.fs, this.lockfile.path);
|
|
832
|
+
const currentVersions = new Set(lockfile ? Object.keys(lockfile.versions) : []);
|
|
833
|
+
const availableVersionsSet = new Set(availableVersionsFromApi);
|
|
834
|
+
const versionsToConsider = options?.versions && options.versions.length > 0 ? options.versions.filter((v) => availableVersionsSet.has(v)) : availableVersionsFromApi;
|
|
835
|
+
if (options?.versions && options.versions.length > 0) {
|
|
836
|
+
const invalidVersions = options.versions.filter((v) => !availableVersionsSet.has(v));
|
|
837
|
+
if (invalidVersions.length > 0) {
|
|
838
|
+
if (invalidVersions.length === 1) throw new UCDStoreVersionNotFoundError(invalidVersions[0]);
|
|
839
|
+
throw new UCDStoreGenericError(`Requested versions are not available in API: ${invalidVersions.join(", ")}`);
|
|
840
|
+
}
|
|
841
|
+
}
|
|
842
|
+
const added = versionsToConsider.filter((v) => !currentVersions.has(v));
|
|
843
|
+
const unchanged = Array.from(currentVersions).filter((v) => {
|
|
844
|
+
if (removeUnavailable && !availableVersionsSet.has(v)) return false;
|
|
845
|
+
if (!removeUnavailable) return true;
|
|
846
|
+
return availableVersionsSet.has(v);
|
|
847
|
+
});
|
|
848
|
+
let removed = [];
|
|
849
|
+
let finalVersions;
|
|
850
|
+
if (removeUnavailable) {
|
|
851
|
+
removed = Array.from(currentVersions).filter((v) => !availableVersionsSet.has(v));
|
|
852
|
+
if (options?.versions && options.versions.length > 0) finalVersions = [...new Set([...Array.from(currentVersions).filter((v) => availableVersionsSet.has(v)), ...versionsToConsider])];
|
|
853
|
+
else finalVersions = availableVersionsFromApi;
|
|
854
|
+
} else finalVersions = [...new Set([...currentVersions, ...added])];
|
|
855
|
+
debug$4?.(`Lockfile update: ${added.length} added, ${removed.length} removed, ${unchanged.length} unchanged`);
|
|
856
|
+
if (lockfile || added.length > 0 || removed.length > 0) {
|
|
857
|
+
const { extractFilterPatterns } = await import("./context-DfX5D4Fb.mjs").then((n) => n.t);
|
|
858
|
+
const filters = extractFilterPatterns(this.filter) ?? lockfile?.filters;
|
|
859
|
+
const now = /* @__PURE__ */ new Date();
|
|
860
|
+
await writeLockfile(this.fs, this.lockfile.path, {
|
|
861
|
+
lockfileVersion: 1,
|
|
862
|
+
createdAt: lockfile?.createdAt ?? now,
|
|
863
|
+
updatedAt: now,
|
|
864
|
+
versions: Object.fromEntries(finalVersions.map((v) => {
|
|
865
|
+
return [v, lockfile?.versions[v] ?? {
|
|
866
|
+
path: `${v}/snapshot.json`,
|
|
867
|
+
fileCount: 0,
|
|
868
|
+
totalSize: 0,
|
|
869
|
+
createdAt: now,
|
|
870
|
+
updatedAt: now
|
|
871
|
+
}];
|
|
872
|
+
})),
|
|
873
|
+
filters
|
|
874
|
+
});
|
|
875
|
+
}
|
|
876
|
+
let versionsToSync;
|
|
877
|
+
if (options?.versions) versionsToSync = options.versions;
|
|
878
|
+
else if (force) versionsToSync = finalVersions;
|
|
879
|
+
else versionsToSync = finalVersions.filter((v) => {
|
|
880
|
+
const entry = lockfile?.versions[v];
|
|
881
|
+
return !entry || entry.fileCount === 0;
|
|
882
|
+
});
|
|
883
|
+
debug$4?.(`Determining versions to sync: ${versionsToSync.join(", ")}`);
|
|
884
|
+
debug$4?.(`Validating versions to sync: ${versionsToSync.join(", ")}`);
|
|
885
|
+
const emptyMirrorReport = {
|
|
886
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
887
|
+
versions: /* @__PURE__ */ new Map(),
|
|
888
|
+
summary: createEmptySummary(0)
|
|
889
|
+
};
|
|
890
|
+
if (versionsToSync.length === 0) {
|
|
891
|
+
debug$4?.("No versions to sync", { versionsToSync });
|
|
892
|
+
const syncDuration = Date.now() - startTime;
|
|
893
|
+
return {
|
|
894
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
895
|
+
added,
|
|
896
|
+
removed,
|
|
897
|
+
unchanged,
|
|
898
|
+
versions: finalVersions,
|
|
899
|
+
mirrorReport: emptyMirrorReport,
|
|
900
|
+
removedFiles: /* @__PURE__ */ new Map(),
|
|
901
|
+
summary: createEmptySummary(syncDuration)
|
|
902
|
+
};
|
|
903
|
+
}
|
|
904
|
+
debug$4?.(`Starting sync for ${versionsToSync.length} version(s) with concurrency=${concurrency}, force=${force}`);
|
|
905
|
+
const [mirrorReport, mirrorError] = await mirror(this, {
|
|
906
|
+
versions: versionsToSync,
|
|
907
|
+
force,
|
|
908
|
+
concurrency,
|
|
909
|
+
filters: options?.filters
|
|
910
|
+
});
|
|
911
|
+
if (mirrorError) throw mirrorError;
|
|
912
|
+
const removedFiles = /* @__PURE__ */ new Map();
|
|
913
|
+
if (cleanOrphaned) {
|
|
914
|
+
const limit = createConcurrencyLimiter(concurrency);
|
|
915
|
+
for (const version of versionsToSync) {
|
|
916
|
+
let expectedFilesPaths = [];
|
|
917
|
+
const snapshot = await readSnapshotOrUndefined(this.fs, version);
|
|
918
|
+
if (snapshot && snapshot.files) expectedFilesPaths = Object.keys(snapshot.files);
|
|
919
|
+
else expectedFilesPaths = (await this.getExpectedFilePaths(version)).map((f) => f.storePath);
|
|
920
|
+
const filteredExpectedFilesPaths = expectedFilesPaths.filter((filePath) => this.filter(filePath, options?.filters));
|
|
921
|
+
const expectedFilesPathsSet = new Set(filteredExpectedFilesPaths);
|
|
922
|
+
const [actualFilesPaths, listFilesError] = await listFiles(this, version, {
|
|
923
|
+
allowApi: false,
|
|
924
|
+
filters: options?.filters
|
|
925
|
+
});
|
|
926
|
+
if (listFilesError) {
|
|
927
|
+
debug$4?.(`Failed to list files for version ${version} when checking orphaned files:`, listFilesError);
|
|
928
|
+
continue;
|
|
929
|
+
}
|
|
930
|
+
debug$4?.("Actual files for version %s: %O", version, actualFilesPaths);
|
|
931
|
+
debug$4?.("Expected files for version %s: %O", version, filteredExpectedFilesPaths);
|
|
932
|
+
const orphanedFiles = actualFilesPaths.filter((filePath) => !filePath.endsWith("snapshot.json")).filter((filePath) => !expectedFilesPathsSet.has(filePath));
|
|
933
|
+
debug$4?.("Orphaned files for version %s: %O", version, orphanedFiles);
|
|
934
|
+
if (orphanedFiles.length > 0) {
|
|
935
|
+
removedFiles.set(version, []);
|
|
936
|
+
await Promise.all(orphanedFiles.map((filePath) => limit(async () => {
|
|
937
|
+
try {
|
|
938
|
+
if (await this.fs.exists(filePath)) {
|
|
939
|
+
await this.fs.rm(filePath);
|
|
940
|
+
removedFiles.get(version).push({
|
|
941
|
+
name: patheBasename(filePath),
|
|
942
|
+
filePath: prependLeadingSlash(filePath)
|
|
943
|
+
});
|
|
944
|
+
debug$4?.(`Removed orphaned file: ${filePath} for version ${version}`);
|
|
945
|
+
}
|
|
946
|
+
} catch (err) {
|
|
947
|
+
debug$4?.(`Failed to remove orphaned file ${filePath} for version ${version}:`, err);
|
|
948
|
+
}
|
|
949
|
+
})));
|
|
950
|
+
}
|
|
951
|
+
}
|
|
952
|
+
}
|
|
953
|
+
const syncDuration = Date.now() - startTime;
|
|
954
|
+
const summary = mirrorReport?.summary ?? createEmptySummary(syncDuration);
|
|
955
|
+
summary.duration = syncDuration;
|
|
956
|
+
return {
|
|
957
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
958
|
+
added,
|
|
959
|
+
removed,
|
|
960
|
+
unchanged,
|
|
961
|
+
versions: finalVersions,
|
|
962
|
+
mirrorReport: mirrorReport ?? emptyMirrorReport,
|
|
963
|
+
removedFiles,
|
|
964
|
+
summary
|
|
965
|
+
};
|
|
966
|
+
});
|
|
967
|
+
}
|
|
968
|
+
function sync(thisOrContext, options) {
|
|
969
|
+
if (isUCDStoreInternalContext(thisOrContext)) return _sync.call(thisOrContext, options);
|
|
970
|
+
return _sync.call(this, thisOrContext);
|
|
971
|
+
}
|
|
972
|
+
async function getVersionsFromApi(context) {
|
|
973
|
+
const configResult = await context.client.config.get();
|
|
974
|
+
let availableVersionsFromApi = [];
|
|
975
|
+
if (configResult.error || !configResult.data) throw new Error("Failed to fetch versions from API");
|
|
976
|
+
else availableVersionsFromApi = configResult.data.versions;
|
|
977
|
+
return availableVersionsFromApi;
|
|
978
|
+
}
|
|
979
|
+
|
|
980
|
+
//#endregion
|
|
981
|
+
//#region src/utils/lockfile.ts
|
|
982
|
+
const debug$3 = createDebugger("ucdjs:ucd-store:init-lockfile");
|
|
983
|
+
/**
|
|
984
|
+
* Initializes a new store lockfile by validating versions against the API
|
|
985
|
+
* and creating the initial lockfile (if the bridge supports writing).
|
|
986
|
+
*
|
|
987
|
+
* This only creates the lockfile structure - it does NOT download any files.
|
|
988
|
+
* Use `mirror()` to download the actual Unicode data files.
|
|
989
|
+
*
|
|
990
|
+
* @param {InternalUCDStoreContext} context - The internal store context
|
|
991
|
+
* @throws {UCDStoreGenericError} If API fetch fails or versions are invalid
|
|
992
|
+
*/
|
|
993
|
+
async function initLockfile(context) {
|
|
994
|
+
const { fs, filter } = context;
|
|
995
|
+
const versions = context.versions.resolved;
|
|
996
|
+
debug$3?.("Starting lockfile initialization for versions:", versions);
|
|
997
|
+
const availableVersions = await context.versions.apiVersions();
|
|
998
|
+
if (availableVersions.length === 0) throw new UCDStoreGenericError("Failed to fetch Unicode versions: no versions available from API");
|
|
999
|
+
debug$3?.(`Fetched ${availableVersions.length} available versions from API`);
|
|
1000
|
+
const missingVersions = versions.filter((v) => !availableVersions.includes(v));
|
|
1001
|
+
if (missingVersions.length > 0) {
|
|
1002
|
+
debug$3?.("✗ Validation failed - missing versions:", missingVersions);
|
|
1003
|
+
throw new UCDStoreGenericError(`Some requested versions are not available in the API: ${missingVersions.join(", ")}`);
|
|
1004
|
+
}
|
|
1005
|
+
debug$3?.("✓ All requested versions are available");
|
|
1006
|
+
if (!await fs.exists(".")) {
|
|
1007
|
+
debug$3?.("Creating store root directory");
|
|
1008
|
+
assertCapability(fs, "mkdir");
|
|
1009
|
+
await fs.mkdir(".");
|
|
1010
|
+
} else debug$3?.("Base directory already exists");
|
|
1011
|
+
if (context.lockfile.supports && context.lockfile.path) {
|
|
1012
|
+
debug$3?.(`Writing lockfile to: ${context.lockfile.path}`);
|
|
1013
|
+
const filters = filter ? extractFilterPatterns(filter) : void 0;
|
|
1014
|
+
const now = /* @__PURE__ */ new Date();
|
|
1015
|
+
await writeLockfile(fs, context.lockfile.path, {
|
|
1016
|
+
lockfileVersion: 1,
|
|
1017
|
+
createdAt: now,
|
|
1018
|
+
updatedAt: now,
|
|
1019
|
+
versions: Object.fromEntries(versions.map((v) => [v, {
|
|
1020
|
+
path: `${v}/snapshot.json`,
|
|
1021
|
+
fileCount: 0,
|
|
1022
|
+
totalSize: 0,
|
|
1023
|
+
createdAt: now,
|
|
1024
|
+
updatedAt: now
|
|
1025
|
+
}])),
|
|
1026
|
+
filters
|
|
1027
|
+
});
|
|
1028
|
+
debug$3?.("✓ Lockfile written");
|
|
1029
|
+
} else debug$3?.("Skipping lockfile write - bridge does not support writing");
|
|
1030
|
+
debug$3?.("✓ Lockfile initialization completed successfully");
|
|
1031
|
+
}
|
|
1032
|
+
|
|
1033
|
+
//#endregion
|
|
1034
|
+
//#region src/utils/validate.ts
|
|
1035
|
+
const debug$2 = createDebugger("ucdjs:ucd-store:validate");
|
|
1036
|
+
/**
|
|
1037
|
+
* Validates that the provided versions are available in the API.
|
|
1038
|
+
* This is a pure validation function with no lockfile dependency.
|
|
1039
|
+
*
|
|
1040
|
+
* @param {ValidateVersionsOptions} options - Validation options
|
|
1041
|
+
* @returns {Promise<ValidateVersionsResult>} Validation result
|
|
1042
|
+
* @throws {UCDStoreGenericError} If API fetch fails
|
|
1043
|
+
*/
|
|
1044
|
+
async function validateVersions(options) {
|
|
1045
|
+
const { client, versions } = options;
|
|
1046
|
+
debug$2?.("Starting version validation for:", versions);
|
|
1047
|
+
const apiResponseResult = await client.versions.list();
|
|
1048
|
+
if (apiResponseResult.error) throw new UCDStoreGenericError(`Failed to fetch Unicode versions during validation: ${apiResponseResult.error.message}${apiResponseResult.error.status ? ` (status ${apiResponseResult.error.status})` : ""}`);
|
|
1049
|
+
if (!apiResponseResult.data) throw new UCDStoreGenericError("Failed to fetch Unicode versions during validation: no data returned");
|
|
1050
|
+
const availableVersions = apiResponseResult.data.map(({ version }) => version);
|
|
1051
|
+
debug$2?.(`Fetched ${availableVersions.length} available versions from API`);
|
|
1052
|
+
const validVersions = versions.filter((v) => availableVersions.includes(v));
|
|
1053
|
+
const invalidVersions = versions.filter((v) => !availableVersions.includes(v));
|
|
1054
|
+
const valid = invalidVersions.length === 0;
|
|
1055
|
+
debug$2?.(valid ? "✓ Validation passed" : "✗ Validation failed", {
|
|
1056
|
+
valid: validVersions.length,
|
|
1057
|
+
invalid: invalidVersions.length
|
|
1058
|
+
});
|
|
1059
|
+
if (invalidVersions.length > 0) debug$2?.("Invalid versions:", invalidVersions);
|
|
1060
|
+
return {
|
|
1061
|
+
valid,
|
|
1062
|
+
validatedVersions: versions,
|
|
1063
|
+
availableVersions,
|
|
1064
|
+
validVersions,
|
|
1065
|
+
invalidVersions
|
|
1066
|
+
};
|
|
1067
|
+
}
|
|
1068
|
+
|
|
1069
|
+
//#endregion
|
|
1070
|
+
//#region src/utils/verify.ts
|
|
1071
|
+
const debug$1 = createDebugger("ucdjs:ucd-store:verify");
|
|
1072
|
+
/**
|
|
1073
|
+
* Verifies that store versions are available in the API.
|
|
1074
|
+
*
|
|
1075
|
+
* If the store supports lockfiles and one exists, it verifies the lockfile versions.
|
|
1076
|
+
* Otherwise, it validates the store's configured versions directly against the API.
|
|
1077
|
+
*
|
|
1078
|
+
* @param {InternalUCDStoreContext} context - Internal store context
|
|
1079
|
+
* @returns {Promise<VerifyResult>} Verification result with comparison details
|
|
1080
|
+
* @throws {UCDStoreGenericError} If lockfile read or API fetch fails
|
|
1081
|
+
*/
|
|
1082
|
+
async function verify(context) {
|
|
1083
|
+
const { client, fs } = context;
|
|
1084
|
+
const { supports: supportsLockfile, exists: lockfileExists, path: lockfilePath } = context.lockfile;
|
|
1085
|
+
const versions = context.versions.resolved;
|
|
1086
|
+
if (supportsLockfile && lockfileExists && lockfilePath) {
|
|
1087
|
+
debug$1?.("Starting lockfile verification");
|
|
1088
|
+
const lockfile = await tryOr({
|
|
1089
|
+
try: () => readLockfile(fs, lockfilePath),
|
|
1090
|
+
err: (err) => {
|
|
1091
|
+
throw new UCDStoreGenericError(`Failed to read lockfile at ${lockfilePath}: ${err.message}`);
|
|
1092
|
+
}
|
|
1093
|
+
});
|
|
1094
|
+
const lockfileVersions = Object.keys(lockfile.versions || {});
|
|
1095
|
+
debug$1?.(`Found ${lockfileVersions.length} versions in lockfile:`, lockfileVersions);
|
|
1096
|
+
const validationResult = await validateVersions({
|
|
1097
|
+
client,
|
|
1098
|
+
versions: lockfileVersions
|
|
1099
|
+
});
|
|
1100
|
+
const extraVersions = validationResult.availableVersions.filter((v) => !lockfileVersions.includes(v));
|
|
1101
|
+
debug$1?.(validationResult.valid ? "✓ Lockfile verification passed" : "✗ Lockfile verification failed", {
|
|
1102
|
+
invalid: validationResult.invalidVersions.length,
|
|
1103
|
+
extra: extraVersions.length
|
|
1104
|
+
});
|
|
1105
|
+
return {
|
|
1106
|
+
valid: validationResult.valid,
|
|
1107
|
+
source: "lockfile",
|
|
1108
|
+
verifiedVersions: lockfileVersions,
|
|
1109
|
+
availableVersions: validationResult.availableVersions,
|
|
1110
|
+
invalidVersions: validationResult.invalidVersions,
|
|
1111
|
+
extraVersions
|
|
1112
|
+
};
|
|
1113
|
+
}
|
|
1114
|
+
debug$1?.("Starting store version validation (no lockfile)");
|
|
1115
|
+
const validationResult = await validateVersions({
|
|
1116
|
+
client,
|
|
1117
|
+
versions
|
|
1118
|
+
});
|
|
1119
|
+
debug$1?.(validationResult.valid ? "✓ Version validation passed" : "✗ Version validation failed", { invalid: validationResult.invalidVersions.length });
|
|
1120
|
+
return {
|
|
1121
|
+
valid: validationResult.valid,
|
|
1122
|
+
source: "store",
|
|
1123
|
+
verifiedVersions: versions,
|
|
1124
|
+
availableVersions: validationResult.availableVersions,
|
|
1125
|
+
invalidVersions: validationResult.invalidVersions,
|
|
1126
|
+
extraVersions: []
|
|
1127
|
+
};
|
|
1128
|
+
}
|
|
1129
|
+
|
|
1130
|
+
//#endregion
|
|
1131
|
+
//#region src/store.ts
|
|
1132
|
+
const debug = createDebugger("ucdjs:ucd-store");
|
|
1133
|
+
async function createUCDStore(options) {
|
|
1134
|
+
debug?.("Creating UCD Store with options", options);
|
|
1135
|
+
const { baseUrl, globalFilters, fs: fsFactory, fsOptions, versions, endpointConfig, requireExistingStore, verify: shouldVerify, versionStrategy } = defu(options, {
|
|
1136
|
+
baseUrl: UCDJS_API_BASE_URL,
|
|
1137
|
+
globalFilters: {},
|
|
1138
|
+
versions: [],
|
|
1139
|
+
requireExistingStore: true,
|
|
1140
|
+
verify: true,
|
|
1141
|
+
versionStrategy: "strict"
|
|
1142
|
+
});
|
|
1143
|
+
const filter = createPathFilter(globalFilters);
|
|
1144
|
+
let resolvedEndpointConfig = endpointConfig;
|
|
1145
|
+
let client = options.client;
|
|
1146
|
+
if (!resolvedEndpointConfig && !client) {
|
|
1147
|
+
debug?.("No endpoint config or client provided, will attempt to discover.");
|
|
1148
|
+
resolvedEndpointConfig = await retrieveEndpointConfiguration(baseUrl);
|
|
1149
|
+
}
|
|
1150
|
+
if (!client) {
|
|
1151
|
+
debug?.("No client provided, creating UCD client with resolved endpoint config.");
|
|
1152
|
+
client = createUCDClientWithConfig(baseUrl, resolvedEndpointConfig);
|
|
1153
|
+
}
|
|
1154
|
+
const discoveryContext = {
|
|
1155
|
+
baseUrl,
|
|
1156
|
+
endpointConfig: resolvedEndpointConfig,
|
|
1157
|
+
versions: resolvedEndpointConfig?.versions ?? []
|
|
1158
|
+
};
|
|
1159
|
+
const resolvedFsOptions = typeof fsOptions === "function" ? fsOptions(discoveryContext) : fsOptions;
|
|
1160
|
+
const fs = resolvedFsOptions !== void 0 ? fsFactory(resolvedFsOptions) : fsFactory();
|
|
1161
|
+
const supportsLockfile = fs.optionalCapabilities?.write === true;
|
|
1162
|
+
const lockfilePath = supportsLockfile ? getLockfilePath() : null;
|
|
1163
|
+
debug?.("Lockfile support:", supportsLockfile, "path:", lockfilePath);
|
|
1164
|
+
const configVersions = resolvedEndpointConfig?.versions ?? [];
|
|
1165
|
+
const lockfileExists = supportsLockfile ? await fs.exists(lockfilePath) : false;
|
|
1166
|
+
debug?.("Lockfile exists:", lockfileExists);
|
|
1167
|
+
let storeVersions = versions;
|
|
1168
|
+
const internalContext = createInternalContext({
|
|
1169
|
+
client,
|
|
1170
|
+
filter,
|
|
1171
|
+
fs,
|
|
1172
|
+
lockfile: {
|
|
1173
|
+
supports: supportsLockfile,
|
|
1174
|
+
exists: lockfileExists,
|
|
1175
|
+
path: lockfilePath ?? ""
|
|
1176
|
+
},
|
|
1177
|
+
versions: {
|
|
1178
|
+
userProvided: versions,
|
|
1179
|
+
configFile: configVersions
|
|
1180
|
+
}
|
|
1181
|
+
});
|
|
1182
|
+
const versionsToValidate = versions.length > 0 ? versions : configVersions;
|
|
1183
|
+
if (versionsToValidate.length > 0) {
|
|
1184
|
+
const apiVersions = await internalContext.versions.apiVersions();
|
|
1185
|
+
if (apiVersions.length === 0) debug?.("Warning: Could not fetch API versions for validation, skipping early check");
|
|
1186
|
+
else {
|
|
1187
|
+
const invalidVersions = versionsToValidate.filter((v) => !apiVersions.includes(v));
|
|
1188
|
+
if (invalidVersions.length > 0) throw new UCDStoreGenericError(`${versions.length > 0 ? "Provided" : "Config"} versions are not available in API: ${invalidVersions.join(", ")}. Available versions: ${apiVersions.slice(0, 5).join(", ")}${apiVersions.length > 5 ? "..." : ""}`);
|
|
1189
|
+
debug?.("✓ Early version validation passed");
|
|
1190
|
+
}
|
|
1191
|
+
}
|
|
1192
|
+
if (internalContext.lockfile.exists && internalContext.lockfile.path) {
|
|
1193
|
+
const lockfile = await readLockfileOrUndefined(fs, internalContext.lockfile.path);
|
|
1194
|
+
const lockfileVersions = lockfile ? Object.keys(lockfile.versions) : [];
|
|
1195
|
+
debug?.("Lockfile versions:", lockfileVersions);
|
|
1196
|
+
if (versions.length > 0) storeVersions = await handleVersionConflict(versionStrategy, versions, lockfileVersions, fs, internalContext.lockfile.path, configVersions, filter);
|
|
1197
|
+
else if (lockfileVersions.length > 0) {
|
|
1198
|
+
storeVersions = lockfileVersions;
|
|
1199
|
+
debug?.("Using versions from lockfile:", storeVersions);
|
|
1200
|
+
} else if (configVersions.length) {
|
|
1201
|
+
storeVersions = configVersions;
|
|
1202
|
+
debug?.("Using versions from config:", storeVersions);
|
|
1203
|
+
}
|
|
1204
|
+
internalContext.versions.resolved = storeVersions;
|
|
1205
|
+
internalContext.lockfile.exists = true;
|
|
1206
|
+
}
|
|
1207
|
+
if (!internalContext.lockfile.exists && internalContext.lockfile.supports) {
|
|
1208
|
+
if (requireExistingStore) throw new UCDStoreGenericError(`Store lockfile not found at ${internalContext.lockfile.path}. Initialize the store first or set requireExistingStore: false to create automatically.`);
|
|
1209
|
+
if (!storeVersions.length && configVersions.length) {
|
|
1210
|
+
storeVersions = configVersions;
|
|
1211
|
+
debug?.("Using versions from config for initialization:", storeVersions);
|
|
1212
|
+
}
|
|
1213
|
+
internalContext.versions.resolved = storeVersions;
|
|
1214
|
+
await initLockfile(internalContext);
|
|
1215
|
+
internalContext.lockfile.exists = true;
|
|
1216
|
+
}
|
|
1217
|
+
if (!internalContext.lockfile.supports) {
|
|
1218
|
+
if (!storeVersions.length && configVersions.length) {
|
|
1219
|
+
storeVersions = configVersions;
|
|
1220
|
+
debug?.("Read-only bridge: using versions from config:", storeVersions);
|
|
1221
|
+
}
|
|
1222
|
+
if (!storeVersions.length) throw new UCDStoreGenericError("No versions provided for read-only file system bridge. Provide versions in options or ensure endpoint config is available.");
|
|
1223
|
+
internalContext.versions.resolved = storeVersions;
|
|
1224
|
+
debug?.("Read-only bridge initialized with versions:", storeVersions);
|
|
1225
|
+
}
|
|
1226
|
+
if (shouldVerify && internalContext.versions.resolved.length > 0) {
|
|
1227
|
+
const verifyResult = await verify(internalContext);
|
|
1228
|
+
if (!verifyResult.valid) throw new UCDStoreGenericError(`${verifyResult.source === "lockfile" ? "Lockfile" : "Version"} verification failed: ${verifyResult.invalidVersions.length} version(s) are not available in API: ${verifyResult.invalidVersions.join(", ")}`);
|
|
1229
|
+
debug?.(`✓ Verification passed (source: ${verifyResult.source})`);
|
|
1230
|
+
}
|
|
1231
|
+
const publicContext = createPublicContext(internalContext);
|
|
1232
|
+
return Object.assign(publicContext, {
|
|
1233
|
+
files: {
|
|
1234
|
+
get: getFile.bind(internalContext),
|
|
1235
|
+
list: listFiles.bind(internalContext),
|
|
1236
|
+
tree: getFileTree.bind(internalContext)
|
|
1237
|
+
},
|
|
1238
|
+
mirror: mirror.bind(internalContext),
|
|
1239
|
+
sync: sync.bind(internalContext),
|
|
1240
|
+
analyze: analyze.bind(internalContext),
|
|
1241
|
+
compare: compare.bind(internalContext)
|
|
1242
|
+
});
|
|
1243
|
+
}
|
|
1244
|
+
async function retrieveEndpointConfiguration(baseUrl = UCDJS_API_BASE_URL) {
|
|
1245
|
+
debug?.("Attempting to discover endpoint configuration from", baseUrl);
|
|
1246
|
+
return discoverEndpointsFromConfig(baseUrl).catch((err) => {
|
|
1247
|
+
debug?.("Failed to discover endpoint config, using default:", err);
|
|
1248
|
+
return getDefaultUCDEndpointConfig();
|
|
1249
|
+
});
|
|
1250
|
+
}
|
|
1251
|
+
async function handleVersionConflict(strategy, providedVersions, lockfileVersions, fs, lockfilePath, _configVersions, filter) {
|
|
1252
|
+
const now = /* @__PURE__ */ new Date();
|
|
1253
|
+
switch (strategy) {
|
|
1254
|
+
case "merge": {
|
|
1255
|
+
const mergedVersions = Array.from(new Set([...lockfileVersions, ...providedVersions]));
|
|
1256
|
+
const existing = await readLockfileOrUndefined(fs, lockfilePath);
|
|
1257
|
+
const { writeLockfile } = await import("@ucdjs/lockfile");
|
|
1258
|
+
const { extractFilterPatterns } = await import("./context-DfX5D4Fb.mjs").then((n) => n.t);
|
|
1259
|
+
const filters = filter ? extractFilterPatterns(filter) : existing?.filters;
|
|
1260
|
+
await writeLockfile(fs, lockfilePath, {
|
|
1261
|
+
lockfileVersion: 1,
|
|
1262
|
+
createdAt: existing?.createdAt ?? now,
|
|
1263
|
+
updatedAt: now,
|
|
1264
|
+
versions: Object.fromEntries(mergedVersions.map((v) => {
|
|
1265
|
+
return [v, existing?.versions[v] ?? {
|
|
1266
|
+
path: `${v}/snapshot.json`,
|
|
1267
|
+
fileCount: 0,
|
|
1268
|
+
totalSize: 0,
|
|
1269
|
+
createdAt: now,
|
|
1270
|
+
updatedAt: now
|
|
1271
|
+
}];
|
|
1272
|
+
})),
|
|
1273
|
+
filters
|
|
1274
|
+
});
|
|
1275
|
+
debug?.("Merge mode: combined versions", mergedVersions);
|
|
1276
|
+
return mergedVersions;
|
|
1277
|
+
}
|
|
1278
|
+
case "overwrite": {
|
|
1279
|
+
const existing = await readLockfileOrUndefined(fs, lockfilePath);
|
|
1280
|
+
const { writeLockfile } = await import("@ucdjs/lockfile");
|
|
1281
|
+
const { extractFilterPatterns } = await import("./context-DfX5D4Fb.mjs").then((n) => n.t);
|
|
1282
|
+
const filters = filter ? extractFilterPatterns(filter) : existing?.filters;
|
|
1283
|
+
await writeLockfile(fs, lockfilePath, {
|
|
1284
|
+
lockfileVersion: 1,
|
|
1285
|
+
createdAt: existing?.createdAt ?? now,
|
|
1286
|
+
updatedAt: now,
|
|
1287
|
+
versions: Object.fromEntries(providedVersions.map((v) => {
|
|
1288
|
+
return [v, existing?.versions[v] ?? {
|
|
1289
|
+
path: `${v}/snapshot.json`,
|
|
1290
|
+
fileCount: 0,
|
|
1291
|
+
totalSize: 0,
|
|
1292
|
+
createdAt: now,
|
|
1293
|
+
updatedAt: now
|
|
1294
|
+
}];
|
|
1295
|
+
})),
|
|
1296
|
+
filters
|
|
1297
|
+
});
|
|
1298
|
+
debug?.("Overwrite mode: replaced lockfile with provided versions", providedVersions);
|
|
1299
|
+
return providedVersions;
|
|
1300
|
+
}
|
|
1301
|
+
default:
|
|
1302
|
+
if (!arraysEqual(providedVersions, lockfileVersions)) throw new UCDStoreGenericError(`Version mismatch: lockfile has [${lockfileVersions.join(", ")}], provided [${providedVersions.join(", ")}]. Use versionStrategy: "merge" or "overwrite" to resolve.`);
|
|
1303
|
+
debug?.("Strict mode: versions match lockfile");
|
|
1304
|
+
return lockfileVersions;
|
|
1305
|
+
}
|
|
1306
|
+
}
|
|
1307
|
+
function arraysEqual(a, b) {
|
|
1308
|
+
if (a.length !== b.length) return false;
|
|
1309
|
+
const setA = new Set(a);
|
|
1310
|
+
const setB = new Set(b);
|
|
1311
|
+
if (setA.size !== setB.size) return false;
|
|
1312
|
+
for (const val of setA) if (!setB.has(val)) return false;
|
|
1313
|
+
return true;
|
|
1314
|
+
}
|
|
1315
|
+
|
|
1316
|
+
//#endregion
|
|
1317
|
+
//#region src/factory.ts
|
|
1318
|
+
/**
|
|
1319
|
+
* Creates a UCD store backed by the Node.js FileSystemBridge.
|
|
1320
|
+
* @template BridgeOptionsSchema extends z.ZodType
|
|
1321
|
+
* @param {NodeUCDStoreOptions<BridgeOptionsSchema>} [options] Store options; provide basePath to set the filesystem root.
|
|
1322
|
+
* @returns {Promise<UCDStore>} A ready-to-use store instance.
|
|
1323
|
+
* @throws {Error} If the Node.js FileSystemBridge could not be loaded.
|
|
1324
|
+
*/
|
|
1325
|
+
async function createNodeUCDStore(options = {}) {
|
|
1326
|
+
const nodeFs = await import("@ucdjs/fs-bridge/bridges/node").then((m) => m.default);
|
|
1327
|
+
if (!nodeFs) throw new Error("Node.js FileSystemBridge could not be loaded");
|
|
1328
|
+
const { basePath, ...storeOptions } = options;
|
|
1329
|
+
const resolvedBasePath = basePath ? patheResolve(basePath) : patheResolve("./");
|
|
1330
|
+
return createUCDStore({
|
|
1331
|
+
...storeOptions,
|
|
1332
|
+
fs: nodeFs,
|
|
1333
|
+
fsOptions: { basePath: resolvedBasePath }
|
|
1334
|
+
});
|
|
1335
|
+
}
|
|
1336
|
+
/**
|
|
1337
|
+
* Creates a UCD store backed by the HTTP FileSystemBridge.
|
|
1338
|
+
* @template BridgeOptionsSchema extends z.ZodType
|
|
1339
|
+
* @param {HTTPUCDStoreOptions<BridgeOptionsSchema>} [options] Store options; provide baseUrl to override the default.
|
|
1340
|
+
* @returns {Promise<UCDStore>} A ready-to-use store instance.
|
|
1341
|
+
* @throws {Error} If the HTTP FileSystemBridge could not be loaded.
|
|
1342
|
+
*/
|
|
1343
|
+
async function createHTTPUCDStore(options = {}) {
|
|
1344
|
+
const httpFsBridge = await import("@ucdjs/fs-bridge/bridges/http").then((m) => m.default);
|
|
1345
|
+
if (!httpFsBridge) throw new Error("HTTP FileSystemBridge could not be loaded");
|
|
1346
|
+
const { bridgeBaseUrl, ...storeOptions } = options;
|
|
1347
|
+
const resolvedBridgeBaseUrl = bridgeBaseUrl ?? UCDJS_STORE_BASE_URL;
|
|
1348
|
+
return createUCDStore({
|
|
1349
|
+
...storeOptions,
|
|
1350
|
+
fs: httpFsBridge,
|
|
1351
|
+
fsOptions: { baseUrl: resolvedBridgeBaseUrl }
|
|
1352
|
+
});
|
|
1353
|
+
}
|
|
1354
|
+
|
|
1355
|
+
//#endregion
|
|
1356
|
+
export { UCDStoreBaseError, UCDStoreBridgeUnsupportedOperation, UCDStoreFileNotFoundError, UCDStoreGenericError, UCDStoreVersionNotFoundError, createHTTPUCDStore, createNodeUCDStore, createUCDStore, validateVersions };
|