@nmakarov/cli-toolkit 0.16.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli-runner.cjs +5006 -0
- package/dist/cli-runner.cjs.map +1 -0
- package/dist/cli-runner.js +4989 -0
- package/dist/cli-runner.js.map +1 -0
- package/dist/filedatabase.cjs +94 -13
- package/dist/filedatabase.cjs.map +1 -1
- package/dist/filedatabase.js +94 -13
- package/dist/filedatabase.js.map +1 -1
- package/dist/http-client2.cjs +1369 -9
- package/dist/http-client2.cjs.map +1 -1
- package/dist/http-client2.js +1359 -9
- package/dist/http-client2.js.map +1 -1
- package/dist/index.cjs +1244 -21
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +1224 -20
- package/dist/index.js.map +1 -1
- package/dist/init.cjs +9 -6
- package/dist/init.cjs.map +1 -1
- package/dist/init.js +9 -6
- package/dist/init.js.map +1 -1
- package/dist/logger.cjs +9 -6
- package/dist/logger.cjs.map +1 -1
- package/dist/logger.js +9 -6
- package/dist/logger.js.map +1 -1
- package/dist/mock-server.cjs +192 -343
- package/dist/mock-server.cjs.map +1 -1
- package/dist/mock-server.js +190 -343
- package/dist/mock-server.js.map +1 -1
- package/dist/tasks.cjs +2295 -0
- package/dist/tasks.cjs.map +1 -0
- package/dist/tasks.js +2240 -0
- package/dist/tasks.js.map +1 -0
- package/dist/utils.cjs +15 -2
- package/dist/utils.cjs.map +1 -1
- package/dist/utils.js +12 -1
- package/dist/utils.js.map +1 -1
- package/package.json +11 -3
package/dist/http-client2.js
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
// src/http-client2/index.ts
|
|
2
|
+
import path4 from "path";
|
|
3
|
+
|
|
1
4
|
// src/errors.ts
|
|
2
5
|
var FrameworkError = class extends Error {
|
|
3
6
|
constructor(message) {
|
|
@@ -5,6 +8,12 @@ var FrameworkError = class extends Error {
|
|
|
5
8
|
this.name = "FrameworkError";
|
|
6
9
|
}
|
|
7
10
|
};
|
|
11
|
+
var ParamError = class extends FrameworkError {
|
|
12
|
+
constructor(message) {
|
|
13
|
+
super(message);
|
|
14
|
+
this.name = "ParamError";
|
|
15
|
+
}
|
|
16
|
+
};
|
|
8
17
|
var HttpClientError = class extends FrameworkError {
|
|
9
18
|
constructor(message, cause) {
|
|
10
19
|
super(message);
|
|
@@ -12,6 +21,12 @@ var HttpClientError = class extends FrameworkError {
|
|
|
12
21
|
this.name = "HttpClientError";
|
|
13
22
|
}
|
|
14
23
|
};
|
|
24
|
+
var FileDatabaseError = class extends FrameworkError {
|
|
25
|
+
constructor(message) {
|
|
26
|
+
super(message);
|
|
27
|
+
this.name = "FileDatabaseError";
|
|
28
|
+
}
|
|
29
|
+
};
|
|
15
30
|
|
|
16
31
|
// src/http-client2/errors.ts
|
|
17
32
|
function classifyError(error) {
|
|
@@ -113,6 +128,1201 @@ function shouldRetryError(classification) {
|
|
|
113
128
|
return classification.retryable;
|
|
114
129
|
}
|
|
115
130
|
|
|
131
|
+
// src/filedatabase/index.ts
|
|
132
|
+
import fs3 from "fs";
|
|
133
|
+
import path3 from "path";
|
|
134
|
+
|
|
135
|
+
// src/utils/os-utils.ts
|
|
136
|
+
import fs from "fs";
|
|
137
|
+
import path from "path";
|
|
138
|
+
import { execSync } from "child_process";
|
|
139
|
+
function getFreeDiskSpace(targetPath) {
|
|
140
|
+
try {
|
|
141
|
+
let pathToCheck = targetPath;
|
|
142
|
+
if (!fs.existsSync(targetPath)) {
|
|
143
|
+
const parentDir = path.dirname(targetPath);
|
|
144
|
+
if (fs.existsSync(parentDir)) {
|
|
145
|
+
pathToCheck = parentDir;
|
|
146
|
+
} else {
|
|
147
|
+
pathToCheck = process.platform === "win32" ? "C:\\" : "/";
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
if (process.platform === "win32") {
|
|
151
|
+
return null;
|
|
152
|
+
} else {
|
|
153
|
+
const stdout = execSync(`df -k "${pathToCheck}"`, { encoding: "utf8" });
|
|
154
|
+
const lines = stdout.trim().split("\n");
|
|
155
|
+
const parts = lines[1].split(/\s+/);
|
|
156
|
+
const freeKb = parseInt(parts[3], 10);
|
|
157
|
+
return freeKb * 1024;
|
|
158
|
+
}
|
|
159
|
+
} catch (error) {
|
|
160
|
+
return null;
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
// src/utils/fs-utils.ts
|
|
165
|
+
import fs2 from "fs";
|
|
166
|
+
import path2 from "path";
|
|
167
|
+
async function ensurePath(...pathParts) {
|
|
168
|
+
const fullPath = path2.resolve(...pathParts);
|
|
169
|
+
if (!fs2.existsSync(fullPath)) {
|
|
170
|
+
await fs2.promises.mkdir(fullPath, { recursive: true });
|
|
171
|
+
}
|
|
172
|
+
return fullPath;
|
|
173
|
+
}
|
|
174
|
+
function getFileExtension(dataType) {
|
|
175
|
+
switch (dataType) {
|
|
176
|
+
case "json-array":
|
|
177
|
+
case "json-object":
|
|
178
|
+
return "json";
|
|
179
|
+
case "text":
|
|
180
|
+
return "txt";
|
|
181
|
+
case "xml":
|
|
182
|
+
return "xml";
|
|
183
|
+
default:
|
|
184
|
+
return "json";
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
// src/utils/format-utils.ts
|
|
189
|
+
function bytesToHumanReadable(bytes) {
|
|
190
|
+
if (bytes === 0) return "0 B";
|
|
191
|
+
const k = 1024;
|
|
192
|
+
const sizes = ["B", "KB", "MB", "GB", "TB", "PB"];
|
|
193
|
+
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
|
194
|
+
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + " " + sizes[i];
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
// src/utils/date-utils.ts
|
|
198
|
+
function isTimestampFolder(folderName) {
|
|
199
|
+
const isoRegex = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(Z|\.\d{3}Z)$/;
|
|
200
|
+
if (!isoRegex.test(folderName)) {
|
|
201
|
+
return false;
|
|
202
|
+
}
|
|
203
|
+
const date = new Date(folderName);
|
|
204
|
+
return !isNaN(date.getTime()) && date.getTime() > 0;
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
// src/filedatabase/serializers.ts
|
|
208
|
+
function detectDataType(data) {
|
|
209
|
+
if (Array.isArray(data)) {
|
|
210
|
+
return "json-array";
|
|
211
|
+
} else if (typeof data === "object" && data !== null) {
|
|
212
|
+
return "json-object";
|
|
213
|
+
} else if (typeof data === "string") {
|
|
214
|
+
const trimmed = data.trim();
|
|
215
|
+
if (trimmed.startsWith("<?xml") || trimmed.startsWith("<")) {
|
|
216
|
+
return "xml";
|
|
217
|
+
}
|
|
218
|
+
return "text";
|
|
219
|
+
} else {
|
|
220
|
+
return "text";
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
function serializeData(data) {
|
|
224
|
+
const dataType = detectDataType(data);
|
|
225
|
+
if (dataType === "json-array" || dataType === "json-object") {
|
|
226
|
+
return JSON.stringify(data, null, 4);
|
|
227
|
+
} else {
|
|
228
|
+
return String(data);
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
function deserializeData(rawData, dataType) {
|
|
232
|
+
if (dataType === "json-array" || dataType === "json-object") {
|
|
233
|
+
return JSON.parse(rawData);
|
|
234
|
+
} else {
|
|
235
|
+
return rawData;
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
// src/filedatabase/index.ts
|
|
240
|
+
var FileDatabase = class _FileDatabase {
|
|
241
|
+
basePath;
|
|
242
|
+
namespace;
|
|
243
|
+
tableName = null;
|
|
244
|
+
versioned;
|
|
245
|
+
maxVersions;
|
|
246
|
+
pageSize;
|
|
247
|
+
useMetadata;
|
|
248
|
+
freeSpaceThreshold;
|
|
249
|
+
logger;
|
|
250
|
+
// Current operation state
|
|
251
|
+
currentVersion = null;
|
|
252
|
+
currentVersionFolder = null;
|
|
253
|
+
currentFileNumber = 0;
|
|
254
|
+
currentRecord = 0;
|
|
255
|
+
hasReadFirstPage = false;
|
|
256
|
+
lastFileData = null;
|
|
257
|
+
metadata;
|
|
258
|
+
// Synopsis calculation functions
|
|
259
|
+
fileSynopsisFunction = null;
|
|
260
|
+
versionSynopsisFunction = null;
|
|
261
|
+
/**
|
|
262
|
+
* Constructor - accepts context as first parameter (new pattern)
|
|
263
|
+
* or config object (legacy pattern for backward compatibility)
|
|
264
|
+
*/
|
|
265
|
+
constructor(contextOrConfig, options) {
|
|
266
|
+
let config;
|
|
267
|
+
if (contextOrConfig && typeof contextOrConfig === "object" && "params" in contextOrConfig) {
|
|
268
|
+
const context = contextOrConfig;
|
|
269
|
+
const opts = options || {};
|
|
270
|
+
const defs = {
|
|
271
|
+
basePath: "string default ./data",
|
|
272
|
+
namespace: "string default default",
|
|
273
|
+
tableName: "string",
|
|
274
|
+
maxVersions: "number default 5",
|
|
275
|
+
pageSize: "number default 5000"
|
|
276
|
+
};
|
|
277
|
+
const discovered = context.params.getAllForModule(defs);
|
|
278
|
+
config = { ...discovered, ...opts, logger: context.logger };
|
|
279
|
+
} else {
|
|
280
|
+
config = contextOrConfig;
|
|
281
|
+
}
|
|
282
|
+
if (!config.basePath) {
|
|
283
|
+
throw new ParamError("[FileDatabase] basePath is required");
|
|
284
|
+
}
|
|
285
|
+
this.basePath = config.basePath;
|
|
286
|
+
this.namespace = config.namespace || "default";
|
|
287
|
+
this.tableName = config.tableName || null;
|
|
288
|
+
this.versioned = config.versioned ?? true;
|
|
289
|
+
this.maxVersions = config.maxVersions || 5;
|
|
290
|
+
this.pageSize = config.pageSize || 5e3;
|
|
291
|
+
this.useMetadata = config.useMetadata !== false;
|
|
292
|
+
this.freeSpaceThreshold = config.freeSpaceThreshold || 100 * 1024 * 1024;
|
|
293
|
+
this.logger = config.logger || console;
|
|
294
|
+
this.metadata = this.getDefaultMetadata();
|
|
295
|
+
}
|
|
296
|
+
/**
|
|
297
|
+
* Initialize FileDatabase from context and options.
|
|
298
|
+
* Params are read via getAllForModule("filedatabase", defs) for --showUsedParams grouping.
|
|
299
|
+
*/
|
|
300
|
+
static init(context, options) {
|
|
301
|
+
return new _FileDatabase(context, options ?? {});
|
|
302
|
+
}
|
|
303
|
+
/**
|
|
304
|
+
* Get default metadata structure
|
|
305
|
+
*/
|
|
306
|
+
getDefaultMetadata() {
|
|
307
|
+
return {
|
|
308
|
+
version: this.currentVersion || null,
|
|
309
|
+
files: [],
|
|
310
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
311
|
+
modifiedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
312
|
+
totalRecords: 0,
|
|
313
|
+
synopsis: null,
|
|
314
|
+
dataType: null
|
|
315
|
+
};
|
|
316
|
+
}
|
|
317
|
+
/**
|
|
318
|
+
* Get the destination path (basePath/namespace/tableName[/version])
|
|
319
|
+
*/
|
|
320
|
+
getDestinationPath(version) {
|
|
321
|
+
const errors = ["basePath", "namespace", "tableName"].filter((prop) => !this[prop]).map((prop) => `${prop} is not set`);
|
|
322
|
+
if (errors.length) {
|
|
323
|
+
throw new FileDatabaseError(`[FileDatabase] ${errors.join("; ")}`);
|
|
324
|
+
}
|
|
325
|
+
let parts = [this.basePath, this.namespace];
|
|
326
|
+
if (this.tableName) {
|
|
327
|
+
parts.push(...this.tableName.split("/"));
|
|
328
|
+
}
|
|
329
|
+
if (this.versioned && version) {
|
|
330
|
+
parts.push(version);
|
|
331
|
+
}
|
|
332
|
+
return path3.resolve(...parts);
|
|
333
|
+
}
|
|
334
|
+
/**
|
|
335
|
+
* Set current version and version folder
|
|
336
|
+
*/
|
|
337
|
+
async setCurrentVersion(version) {
|
|
338
|
+
this.currentVersion = version;
|
|
339
|
+
this.currentVersionFolder = await ensurePath(this.getDestinationPath(), version);
|
|
340
|
+
}
|
|
341
|
+
/**
|
|
342
|
+
* Create a new version folder with comprehensive timestamp logic
|
|
343
|
+
* Only works in versioned mode
|
|
344
|
+
*/
|
|
345
|
+
async makeNewVersion() {
|
|
346
|
+
if (!this.versioned) {
|
|
347
|
+
throw new FileDatabaseError("makeNewVersion() only works in versioned mode");
|
|
348
|
+
}
|
|
349
|
+
this.metadata = this.getDefaultMetadata();
|
|
350
|
+
const existingVersions = await this.getVersions();
|
|
351
|
+
let versionName;
|
|
352
|
+
if (existingVersions.length > 0) {
|
|
353
|
+
const maxTimestamp = existingVersions.reduce((max, version) => {
|
|
354
|
+
const versionDate = new Date(version.replace("Z", ""));
|
|
355
|
+
const maxDate2 = new Date(max.replace("Z", ""));
|
|
356
|
+
return versionDate > maxDate2 ? version : max;
|
|
357
|
+
});
|
|
358
|
+
const maxDate = new Date(maxTimestamp.replace("Z", ""));
|
|
359
|
+
const nextDate = new Date(maxDate.getTime() + 1e3);
|
|
360
|
+
versionName = nextDate.toISOString().split(".")[0] + "Z";
|
|
361
|
+
} else {
|
|
362
|
+
const now = /* @__PURE__ */ new Date();
|
|
363
|
+
versionName = now.toISOString().split(".")[0] + "Z";
|
|
364
|
+
}
|
|
365
|
+
await this.setCurrentVersion(versionName);
|
|
366
|
+
this.currentFileNumber = 0;
|
|
367
|
+
const versions = await this.getVersions();
|
|
368
|
+
while (versions.length > this.maxVersions) {
|
|
369
|
+
const versionToDelete = path3.resolve(this.getDestinationPath(), versions.shift());
|
|
370
|
+
this.logger.silly?.(`[FileDatabase] Deleting old version: ${versionToDelete}`);
|
|
371
|
+
await fs3.promises.rm(versionToDelete, { recursive: true, force: true });
|
|
372
|
+
}
|
|
373
|
+
return versionName;
|
|
374
|
+
}
|
|
375
|
+
/**
|
|
376
|
+
* Get list of all versions (sorted chronologically)
|
|
377
|
+
* Only works in versioned mode
|
|
378
|
+
*/
|
|
379
|
+
async getVersions() {
|
|
380
|
+
if (!this.versioned) {
|
|
381
|
+
return [];
|
|
382
|
+
}
|
|
383
|
+
const destPath = this.getDestinationPath();
|
|
384
|
+
try {
|
|
385
|
+
await ensurePath(destPath);
|
|
386
|
+
const items = await fs3.promises.readdir(destPath);
|
|
387
|
+
const versions = items.filter((item) => {
|
|
388
|
+
const itemPath = path3.join(destPath, item);
|
|
389
|
+
const stat = fs3.statSync(itemPath);
|
|
390
|
+
return stat.isDirectory() && isTimestampFolder(item);
|
|
391
|
+
});
|
|
392
|
+
return versions.sort();
|
|
393
|
+
} catch (error) {
|
|
394
|
+
return [];
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
/**
|
|
398
|
+
* Get the latest version (most recent timestamp)
|
|
399
|
+
* Only works in versioned mode
|
|
400
|
+
* @returns Latest version string or null if no versions
|
|
401
|
+
*/
|
|
402
|
+
async getLatestVersion() {
|
|
403
|
+
if (!this.versioned) {
|
|
404
|
+
throw new FileDatabaseError("getLatestVersion() only works in versioned mode");
|
|
405
|
+
}
|
|
406
|
+
const versions = await this.getVersions();
|
|
407
|
+
if (versions.length === 0) {
|
|
408
|
+
return null;
|
|
409
|
+
}
|
|
410
|
+
return versions[versions.length - 1];
|
|
411
|
+
}
|
|
412
|
+
/**
|
|
413
|
+
* Check if any data exists in this table
|
|
414
|
+
* Works for both versioned and non-versioned modes
|
|
415
|
+
* @returns true if data exists
|
|
416
|
+
*/
|
|
417
|
+
async hasData() {
|
|
418
|
+
const tablePath = this.getDestinationPath();
|
|
419
|
+
if (!fs3.existsSync(tablePath)) {
|
|
420
|
+
return false;
|
|
421
|
+
}
|
|
422
|
+
if (this.versioned) {
|
|
423
|
+
const versions = await this.getVersions();
|
|
424
|
+
return versions.length > 0;
|
|
425
|
+
} else {
|
|
426
|
+
const items = await fs3.promises.readdir(tablePath);
|
|
427
|
+
return items.some(
|
|
428
|
+
(item) => item === "metadata.json" || item.match(/^\d{6}\.(json|txt|xml)$/) || item.endsWith(".json")
|
|
429
|
+
);
|
|
430
|
+
}
|
|
431
|
+
}
|
|
432
|
+
/**
|
|
433
|
+
* Auto-detect the data format in this table
|
|
434
|
+
* Used when reading existing data
|
|
435
|
+
* @returns Format detection result
|
|
436
|
+
*/
|
|
437
|
+
async detectDataFormat() {
|
|
438
|
+
const tablePath = this.getDestinationPath();
|
|
439
|
+
if (!fs3.existsSync(tablePath)) {
|
|
440
|
+
return { versioned: false, hasMetadata: false, dataType: null };
|
|
441
|
+
}
|
|
442
|
+
const items = await fs3.promises.readdir(tablePath);
|
|
443
|
+
if (items.includes("metadata.json")) {
|
|
444
|
+
const metadata = JSON.parse(
|
|
445
|
+
await fs3.promises.readFile(path3.join(tablePath, "metadata.json"), "utf8")
|
|
446
|
+
);
|
|
447
|
+
return {
|
|
448
|
+
versioned: false,
|
|
449
|
+
hasMetadata: true,
|
|
450
|
+
dataType: metadata.dataType || null
|
|
451
|
+
};
|
|
452
|
+
}
|
|
453
|
+
const versionFolders = items.filter((item) => {
|
|
454
|
+
const itemPath = path3.join(tablePath, item);
|
|
455
|
+
const stat = fs3.statSync(itemPath);
|
|
456
|
+
return stat.isDirectory() && isTimestampFolder(item);
|
|
457
|
+
});
|
|
458
|
+
if (versionFolders.length > 0) {
|
|
459
|
+
const latestVersion = versionFolders.sort().pop();
|
|
460
|
+
const versionMetadataPath = path3.join(tablePath, latestVersion, "metadata.json");
|
|
461
|
+
return {
|
|
462
|
+
versioned: true,
|
|
463
|
+
hasMetadata: fs3.existsSync(versionMetadataPath),
|
|
464
|
+
dataType: null
|
|
465
|
+
};
|
|
466
|
+
}
|
|
467
|
+
const dataFiles = items.filter((f) => f.match(/^\d{6}\.(json|txt|xml)$/));
|
|
468
|
+
if (dataFiles.length > 0) {
|
|
469
|
+
return {
|
|
470
|
+
versioned: false,
|
|
471
|
+
hasMetadata: false,
|
|
472
|
+
dataType: null
|
|
473
|
+
};
|
|
474
|
+
}
|
|
475
|
+
return { versioned: false, hasMetadata: false, dataType: null };
|
|
476
|
+
}
|
|
477
|
+
/**
|
|
478
|
+
* Load metadata from JSON file
|
|
479
|
+
*/
|
|
480
|
+
async loadMetadataJson(version) {
|
|
481
|
+
const metadataFile = path3.join(this.getDestinationPath(), version, "metadata.json");
|
|
482
|
+
if (fs3.existsSync(metadataFile)) {
|
|
483
|
+
try {
|
|
484
|
+
const rawData = await fs3.promises.readFile(metadataFile, "utf8");
|
|
485
|
+
return JSON.parse(rawData);
|
|
486
|
+
} catch (e) {
|
|
487
|
+
throw new FileDatabaseError(`Failed to read metadata for version "${version}": ${e.message}`);
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
return null;
|
|
491
|
+
}
|
|
492
|
+
/**
|
|
493
|
+
* Build metadata by scanning files in a version folder (backward compatibility)
|
|
494
|
+
* Reads all files to get accurate counts - used when synopsis calculation is needed
|
|
495
|
+
*/
|
|
496
|
+
async figureMetadataFromVersionFiles(version) {
|
|
497
|
+
const versionPath = path3.join(this.getDestinationPath(), version);
|
|
498
|
+
if (!fs3.existsSync(versionPath)) {
|
|
499
|
+
return this.getDefaultMetadata();
|
|
500
|
+
}
|
|
501
|
+
const files = (await fs3.promises.readdir(versionPath)).filter((file) => file !== "metadata.json" && !file.startsWith(".")).sort();
|
|
502
|
+
const metadata = this.getDefaultMetadata();
|
|
503
|
+
metadata.version = version;
|
|
504
|
+
metadata.files = [];
|
|
505
|
+
let totalRecords = 0;
|
|
506
|
+
let detectedDataType = null;
|
|
507
|
+
for (let i = 0; i < files.length; i++) {
|
|
508
|
+
const fileName = files[i];
|
|
509
|
+
const filePath = path3.join(versionPath, fileName);
|
|
510
|
+
try {
|
|
511
|
+
const rawData = await fs3.promises.readFile(filePath, "utf8");
|
|
512
|
+
const extension = path3.extname(fileName).toLowerCase();
|
|
513
|
+
let dataType = "text";
|
|
514
|
+
if (extension === ".json") {
|
|
515
|
+
dataType = "json-array";
|
|
516
|
+
} else if (extension === ".xml") {
|
|
517
|
+
dataType = "xml";
|
|
518
|
+
}
|
|
519
|
+
const fileData = deserializeData(rawData, dataType);
|
|
520
|
+
const recordsCount = Array.isArray(fileData) ? fileData.length : 1;
|
|
521
|
+
if (detectedDataType === null) {
|
|
522
|
+
detectedDataType = detectDataType(fileData);
|
|
523
|
+
}
|
|
524
|
+
const fileInfo = {
|
|
525
|
+
number: i + 1,
|
|
526
|
+
recordsCount,
|
|
527
|
+
fileName
|
|
528
|
+
};
|
|
529
|
+
metadata.files.push(fileInfo);
|
|
530
|
+
totalRecords += recordsCount;
|
|
531
|
+
} catch (error) {
|
|
532
|
+
this.logger.error?.(`[FileDatabase] Failed to read file ${fileName}: ${error.message}`);
|
|
533
|
+
}
|
|
534
|
+
}
|
|
535
|
+
metadata.totalRecords = totalRecords;
|
|
536
|
+
metadata.dataType = detectedDataType;
|
|
537
|
+
return metadata;
|
|
538
|
+
}
|
|
539
|
+
/**
|
|
540
|
+
* Build metadata optimized - only reads first and last files
|
|
541
|
+
* Assumes all middle files have the same record count as the first file
|
|
542
|
+
* Much faster for large datasets with many files
|
|
543
|
+
*/
|
|
544
|
+
async buildMetadataOptimized(version) {
|
|
545
|
+
const versionPath = path3.join(this.getDestinationPath(), version);
|
|
546
|
+
if (!fs3.existsSync(versionPath)) {
|
|
547
|
+
return this.getDefaultMetadata();
|
|
548
|
+
}
|
|
549
|
+
const files = (await fs3.promises.readdir(versionPath)).filter((file) => file !== "metadata.json" && !file.startsWith(".")).sort();
|
|
550
|
+
if (files.length === 0) {
|
|
551
|
+
return this.getDefaultMetadata();
|
|
552
|
+
}
|
|
553
|
+
const metadata = this.getDefaultMetadata();
|
|
554
|
+
metadata.version = version;
|
|
555
|
+
metadata.files = files.map((fileName, index) => ({
|
|
556
|
+
number: index + 1,
|
|
557
|
+
recordsCount: 0,
|
|
558
|
+
fileName
|
|
559
|
+
}));
|
|
560
|
+
const firstFile = metadata.files[0];
|
|
561
|
+
const firstFilePath = path3.join(versionPath, firstFile.fileName);
|
|
562
|
+
const firstFileRaw = await fs3.promises.readFile(firstFilePath, "utf8");
|
|
563
|
+
let firstFileData;
|
|
564
|
+
try {
|
|
565
|
+
firstFileData = JSON.parse(firstFileRaw);
|
|
566
|
+
} catch (e) {
|
|
567
|
+
firstFileData = firstFileRaw;
|
|
568
|
+
}
|
|
569
|
+
metadata.dataType = detectDataType(firstFileData);
|
|
570
|
+
if (metadata.dataType === "json-array") {
|
|
571
|
+
const firstFileCount = Array.isArray(firstFileData) ? firstFileData.length : 1;
|
|
572
|
+
firstFile.recordsCount = firstFileCount;
|
|
573
|
+
for (let i = 1; i < metadata.files.length - 1; i++) {
|
|
574
|
+
metadata.files[i].recordsCount = firstFileCount;
|
|
575
|
+
}
|
|
576
|
+
if (files.length > 1) {
|
|
577
|
+
const lastFile = metadata.files[metadata.files.length - 1];
|
|
578
|
+
const lastFilePath = path3.join(versionPath, lastFile.fileName);
|
|
579
|
+
const lastFileRaw = await fs3.promises.readFile(lastFilePath, "utf8");
|
|
580
|
+
const lastFileData = deserializeData(lastFileRaw, metadata.dataType);
|
|
581
|
+
lastFile.recordsCount = Array.isArray(lastFileData) ? lastFileData.length : 1;
|
|
582
|
+
}
|
|
583
|
+
metadata.totalRecords = metadata.files.reduce((sum, file) => sum + file.recordsCount, 0);
|
|
584
|
+
} else {
|
|
585
|
+
metadata.files.forEach((file) => {
|
|
586
|
+
file.recordsCount = 1;
|
|
587
|
+
});
|
|
588
|
+
metadata.totalRecords = files.length;
|
|
589
|
+
}
|
|
590
|
+
return metadata;
|
|
591
|
+
}
|
|
592
|
+
/**
|
|
593
|
+
* Figure out metadata - tries JSON first, then builds from files
|
|
594
|
+
* Uses optimized building when no synopsis calculation is needed
|
|
595
|
+
*/
|
|
596
|
+
async figureMetadata(version, useOptimized = true) {
|
|
597
|
+
if (this.useMetadata) {
|
|
598
|
+
const metadata = await this.loadMetadataJson(version);
|
|
599
|
+
if (metadata) {
|
|
600
|
+
return metadata;
|
|
601
|
+
}
|
|
602
|
+
}
|
|
603
|
+
if (useOptimized && !this.fileSynopsisFunction && !this.versionSynopsisFunction) {
|
|
604
|
+
return await this.buildMetadataOptimized(version);
|
|
605
|
+
}
|
|
606
|
+
return await this.figureMetadataFromVersionFiles(version);
|
|
607
|
+
}
|
|
608
|
+
/**
|
|
609
|
+
* Load version metadata (main entry point for loading)
|
|
610
|
+
*/
|
|
611
|
+
async loadVersionMetadata(version) {
|
|
612
|
+
const metadata = await this.figureMetadata(version);
|
|
613
|
+
this.metadata = metadata;
|
|
614
|
+
return metadata;
|
|
615
|
+
}
|
|
616
|
+
/**
|
|
617
|
+
* Save version metadata to file
|
|
618
|
+
*/
|
|
619
|
+
async saveVersionMetadata(metadata) {
|
|
620
|
+
if (!this.useMetadata) {
|
|
621
|
+
return;
|
|
622
|
+
}
|
|
623
|
+
const metadataToSave = metadata || this.metadata;
|
|
624
|
+
let metadataFile;
|
|
625
|
+
if (this.versioned) {
|
|
626
|
+
if (!this.currentVersion) {
|
|
627
|
+
return;
|
|
628
|
+
}
|
|
629
|
+
metadataFile = path3.join(this.getDestinationPath(), this.currentVersion, "metadata.json");
|
|
630
|
+
} else {
|
|
631
|
+
metadataFile = path3.join(this.getDestinationPath(), "metadata.json");
|
|
632
|
+
}
|
|
633
|
+
await fs3.promises.writeFile(metadataFile, JSON.stringify(metadataToSave, null, 4), "utf8");
|
|
634
|
+
}
|
|
635
|
+
/**
|
|
636
|
+
* Create a new file entry in metadata
|
|
637
|
+
*/
|
|
638
|
+
makeNewFile() {
|
|
639
|
+
this.currentFileNumber = (this.currentFileNumber || 0) + 1;
|
|
640
|
+
const dataType = this.metadata.dataType || "json-array";
|
|
641
|
+
const fileEntry = {
|
|
642
|
+
number: this.currentFileNumber,
|
|
643
|
+
recordsCount: 0,
|
|
644
|
+
fileName: `${this.currentFileNumber.toString().padStart(6, "0")}.${getFileExtension(dataType)}`
|
|
645
|
+
};
|
|
646
|
+
this.metadata.files.push(fileEntry);
|
|
647
|
+
this.lastFileData = null;
|
|
648
|
+
this.logger.silly?.(`[FileDatabase] Created new file: ${fileEntry.fileName}, fileNumber: ${this.currentFileNumber}`);
|
|
649
|
+
}
|
|
650
|
+
/**
|
|
651
|
+
* Figure out what data to write and which file to use (for pagination)
|
|
652
|
+
* @param data - Data to write
|
|
653
|
+
* @param targetFileIndex - Optional index of existing file to overwrite (when customMetadata matches)
|
|
654
|
+
* @param forceNewFile - If true, always create a new file (when customMetadata provided but no match)
|
|
655
|
+
*/
|
|
656
|
+
figureOutDataAndFileToWrite(data, targetFileIndex = null, forceNewFile = false) {
|
|
657
|
+
let dataToWrite;
|
|
658
|
+
let dataLeftOver;
|
|
659
|
+
const incomingDataType = detectDataType(data);
|
|
660
|
+
if (this.metadata.dataType !== incomingDataType) {
|
|
661
|
+
this.metadata.dataType = incomingDataType;
|
|
662
|
+
}
|
|
663
|
+
if (targetFileIndex !== null && targetFileIndex < this.metadata.files.length) {
|
|
664
|
+
const targetFile = this.metadata.files[targetFileIndex];
|
|
665
|
+
if (!Array.isArray(data)) {
|
|
666
|
+
dataToWrite = data;
|
|
667
|
+
dataLeftOver = null;
|
|
668
|
+
return { dataToWrite, dataLeftOver, fileName: targetFile.fileName };
|
|
669
|
+
} else {
|
|
670
|
+
dataToWrite = data.slice(0, this.pageSize);
|
|
671
|
+
dataLeftOver = data.slice(this.pageSize);
|
|
672
|
+
this.lastFileData = dataToWrite;
|
|
673
|
+
return { dataToWrite, dataLeftOver, fileName: targetFile.fileName };
|
|
674
|
+
}
|
|
675
|
+
}
|
|
676
|
+
let newlyCreatedFileIndex = null;
|
|
677
|
+
if (forceNewFile) {
|
|
678
|
+
const filesBeforeCreate = this.metadata.files.length;
|
|
679
|
+
this.makeNewFile();
|
|
680
|
+
newlyCreatedFileIndex = filesBeforeCreate;
|
|
681
|
+
this.logger.silly?.(`[FileDatabase] Creating new file for unique custom metadata combination, fileNumber: ${this.currentFileNumber}`);
|
|
682
|
+
} else if (this.metadata.files.length === 0) {
|
|
683
|
+
this.makeNewFile();
|
|
684
|
+
}
|
|
685
|
+
const lastFile = this.metadata.files[this.metadata.files.length - 1];
|
|
686
|
+
const lastFileRecordsCount = lastFile.recordsCount;
|
|
687
|
+
if (forceNewFile && newlyCreatedFileIndex !== null) {
|
|
688
|
+
const newlyCreatedFile = this.metadata.files[newlyCreatedFileIndex];
|
|
689
|
+
if (newlyCreatedFile && newlyCreatedFile.fileName !== lastFile.fileName) {
|
|
690
|
+
this.logger.warn?.(`[FileDatabase] Warning: Newly created file ${newlyCreatedFile.fileName} doesn't match last file ${lastFile.fileName}`);
|
|
691
|
+
}
|
|
692
|
+
}
|
|
693
|
+
if (!Array.isArray(data) && !forceNewFile) {
|
|
694
|
+
const lastFileExtension = path3.extname(lastFile.fileName);
|
|
695
|
+
const expectedExtension = `.${getFileExtension(incomingDataType)}`;
|
|
696
|
+
if (lastFileExtension !== expectedExtension) {
|
|
697
|
+
if (lastFileRecordsCount > 0) {
|
|
698
|
+
this.makeNewFile();
|
|
699
|
+
} else {
|
|
700
|
+
lastFile.fileName = `${this.currentFileNumber.toString().padStart(6, "0")}.${getFileExtension(incomingDataType)}`;
|
|
701
|
+
}
|
|
702
|
+
}
|
|
703
|
+
} else if (!Array.isArray(data) && forceNewFile) {
|
|
704
|
+
const lastFileExtension = path3.extname(lastFile.fileName);
|
|
705
|
+
const expectedExtension = `.${getFileExtension(incomingDataType)}`;
|
|
706
|
+
if (lastFileExtension !== expectedExtension) {
|
|
707
|
+
lastFile.fileName = `${this.currentFileNumber.toString().padStart(6, "0")}.${getFileExtension(incomingDataType)}`;
|
|
708
|
+
}
|
|
709
|
+
}
|
|
710
|
+
if (Array.isArray(data)) {
|
|
711
|
+
if (forceNewFile) {
|
|
712
|
+
dataToWrite = data.slice(0, this.pageSize);
|
|
713
|
+
dataLeftOver = data.slice(this.pageSize);
|
|
714
|
+
this.lastFileData = dataToWrite;
|
|
715
|
+
} else if (lastFileRecordsCount < this.pageSize) {
|
|
716
|
+
dataToWrite = [...this.lastFileData || [], ...data.slice(0, this.pageSize - lastFileRecordsCount)];
|
|
717
|
+
dataLeftOver = data.slice(this.pageSize - lastFileRecordsCount);
|
|
718
|
+
this.lastFileData = dataToWrite;
|
|
719
|
+
} else {
|
|
720
|
+
this.makeNewFile();
|
|
721
|
+
dataToWrite = data.slice(0, this.pageSize);
|
|
722
|
+
dataLeftOver = data.slice(this.pageSize);
|
|
723
|
+
this.lastFileData = dataToWrite;
|
|
724
|
+
}
|
|
725
|
+
} else {
|
|
726
|
+
dataToWrite = data;
|
|
727
|
+
dataLeftOver = null;
|
|
728
|
+
}
|
|
729
|
+
const fileName = this.metadata.files[this.metadata.files.length - 1].fileName;
|
|
730
|
+
this.logger.silly?.(
|
|
731
|
+
`[FileDatabase] figureOutDataAndFileToWrite: filename=${fileName}, forceNewFile=${forceNewFile}, targetFileIndex=${targetFileIndex}, dataToWrite.length=${Array.isArray(dataToWrite) ? dataToWrite.length : "N/A"}, lastFileRecordsCount=${lastFileRecordsCount}`
|
|
732
|
+
);
|
|
733
|
+
return { dataToWrite, dataLeftOver, fileName };
|
|
734
|
+
}
|
|
735
|
+
/**
|
|
736
|
+
* Calculate file-level synopsis if function is set
|
|
737
|
+
*/
|
|
738
|
+
calculateFileSynopsis(data, fileIndex = this.metadata.files.length - 1) {
|
|
739
|
+
if (!this.fileSynopsisFunction) {
|
|
740
|
+
return;
|
|
741
|
+
}
|
|
742
|
+
const fileInfo = this.metadata.files[fileIndex];
|
|
743
|
+
const enhancedFileInfo = this.fileSynopsisFunction(fileInfo, data);
|
|
744
|
+
this.metadata.files[fileIndex] = enhancedFileInfo;
|
|
745
|
+
}
|
|
746
|
+
/**
|
|
747
|
+
* Calculate version-level synopsis if function is set
|
|
748
|
+
*/
|
|
749
|
+
calculateVersionSynopsis() {
|
|
750
|
+
if (!this.versionSynopsisFunction) {
|
|
751
|
+
return;
|
|
752
|
+
}
|
|
753
|
+
const enhancedMetadata = this.versionSynopsisFunction(this.metadata);
|
|
754
|
+
this.metadata = enhancedMetadata;
|
|
755
|
+
}
|
|
756
|
+
/**
|
|
757
|
+
* Update metadata after writing data
|
|
758
|
+
*/
|
|
759
|
+
updateMetadata(dataToWrite, fileName, customMetadata) {
|
|
760
|
+
let currentFile;
|
|
761
|
+
if (fileName) {
|
|
762
|
+
const foundFile = this.metadata.files.find((file) => file.fileName === fileName);
|
|
763
|
+
if (!foundFile) {
|
|
764
|
+
this.logger.warn?.(`[FileDatabase] File ${fileName} not found in metadata, using last file`);
|
|
765
|
+
currentFile = this.metadata.files[this.metadata.files.length - 1];
|
|
766
|
+
} else {
|
|
767
|
+
currentFile = foundFile;
|
|
768
|
+
}
|
|
769
|
+
} else {
|
|
770
|
+
currentFile = this.metadata.files[this.metadata.files.length - 1];
|
|
771
|
+
}
|
|
772
|
+
const recordsCount = Array.isArray(dataToWrite) ? dataToWrite.length : 1;
|
|
773
|
+
currentFile.recordsCount = recordsCount;
|
|
774
|
+
if (customMetadata) {
|
|
775
|
+
Object.assign(currentFile, customMetadata);
|
|
776
|
+
}
|
|
777
|
+
const fileIndex = this.metadata.files.indexOf(currentFile);
|
|
778
|
+
if (fileIndex !== -1) {
|
|
779
|
+
this.calculateFileSynopsis(dataToWrite, fileIndex);
|
|
780
|
+
}
|
|
781
|
+
this.metadata.version = this.currentVersion;
|
|
782
|
+
this.metadata.modifiedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
783
|
+
this.metadata.dataType = detectDataType(dataToWrite);
|
|
784
|
+
this.metadata.totalRecords = this.metadata.files.reduce((sum, file) => sum + (file.recordsCount || 0), 0);
|
|
785
|
+
this.logger.silly?.(
|
|
786
|
+
`[FileDatabase] Updated metadata for file ${currentFile.fileName}: recordsCount=${recordsCount}, totalRecords=${this.metadata.totalRecords}`
|
|
787
|
+
);
|
|
788
|
+
}
|
|
789
|
+
/**
|
|
790
|
+
* Safe write with disk space check
|
|
791
|
+
*/
|
|
792
|
+
async safeWrite(filePath, data) {
|
|
793
|
+
const serializedData = serializeData(data);
|
|
794
|
+
const dir = path3.dirname(filePath);
|
|
795
|
+
const requiredBytes = Buffer.byteLength(serializedData, "utf8");
|
|
796
|
+
const freeBytes = getFreeDiskSpace(dir);
|
|
797
|
+
if (freeBytes !== null) {
|
|
798
|
+
if (freeBytes < requiredBytes) {
|
|
799
|
+
throw new FileDatabaseError(
|
|
800
|
+
`Not enough disk space. Required: ${bytesToHumanReadable(requiredBytes)}, Free: ${bytesToHumanReadable(freeBytes)}`
|
|
801
|
+
);
|
|
802
|
+
}
|
|
803
|
+
if (freeBytes < this.freeSpaceThreshold) {
|
|
804
|
+
this.logger.warn?.(`Low disk space warning: only ${bytesToHumanReadable(freeBytes)} left`);
|
|
805
|
+
}
|
|
806
|
+
}
|
|
807
|
+
try {
|
|
808
|
+
await fs3.promises.writeFile(filePath, serializedData, "utf8");
|
|
809
|
+
this.logger.silly?.(`[FileDatabase] Wrote ${bytesToHumanReadable(requiredBytes)} to ${filePath}`);
|
|
810
|
+
} catch (error) {
|
|
811
|
+
throw new FileDatabaseError(`Failed to write file ${filePath}: ${error.message}`);
|
|
812
|
+
}
|
|
813
|
+
}
|
|
814
|
+
/**
|
|
815
|
+
* Prepare the instance for read or write operations
|
|
816
|
+
* This discovers state and sets up internal members based on mode and current data
|
|
817
|
+
*/
|
|
818
|
+
async prepare({ write, read, version }) {
|
|
819
|
+
if (write) {
|
|
820
|
+
if (this.versioned) {
|
|
821
|
+
if (this.currentVersion === null) {
|
|
822
|
+
await this.makeNewVersion();
|
|
823
|
+
this.metadata = this.getDefaultMetadata();
|
|
824
|
+
this.metadata.version = this.currentVersion;
|
|
825
|
+
this.makeNewFile();
|
|
826
|
+
} else {
|
|
827
|
+
if (!this.metadata.files.length) {
|
|
828
|
+
this.metadata = await this.figureMetadata(this.currentVersion);
|
|
829
|
+
if (this.metadata.files && this.metadata.files.length > 0) {
|
|
830
|
+
this.currentFileNumber = Math.max(...this.metadata.files.map((f) => f.number || 0));
|
|
831
|
+
} else {
|
|
832
|
+
this.currentFileNumber = 0;
|
|
833
|
+
}
|
|
834
|
+
}
|
|
835
|
+
}
|
|
836
|
+
} else {
|
|
837
|
+
await ensurePath(this.getDestinationPath());
|
|
838
|
+
if (this.useMetadata === true) {
|
|
839
|
+
const metadataPath = path3.join(this.getDestinationPath(), "metadata.json");
|
|
840
|
+
if (fs3.existsSync(metadataPath)) {
|
|
841
|
+
try {
|
|
842
|
+
const rawData = await fs3.promises.readFile(metadataPath, "utf8");
|
|
843
|
+
this.metadata = JSON.parse(rawData);
|
|
844
|
+
if (this.metadata.files && this.metadata.files.length > 0) {
|
|
845
|
+
this.currentFileNumber = Math.max(...this.metadata.files.map((f) => f.number || 0));
|
|
846
|
+
} else {
|
|
847
|
+
this.currentFileNumber = 0;
|
|
848
|
+
}
|
|
849
|
+
} catch (e) {
|
|
850
|
+
this.metadata = this.getDefaultMetadata();
|
|
851
|
+
this.currentFileNumber = 0;
|
|
852
|
+
}
|
|
853
|
+
} else {
|
|
854
|
+
this.metadata = this.getDefaultMetadata();
|
|
855
|
+
this.currentFileNumber = 0;
|
|
856
|
+
}
|
|
857
|
+
} else {
|
|
858
|
+
this.metadata = this.getDefaultMetadata();
|
|
859
|
+
this.currentFileNumber = 0;
|
|
860
|
+
}
|
|
861
|
+
}
|
|
862
|
+
} else if (read) {
|
|
863
|
+
if (this.versioned) {
|
|
864
|
+
const versions = await this.getVersions();
|
|
865
|
+
if (versions.length === 0) {
|
|
866
|
+
throw new FileDatabaseError("[FileDatabase] No versions found, cannot read");
|
|
867
|
+
}
|
|
868
|
+
if (version) {
|
|
869
|
+
if (!versions.includes(version)) {
|
|
870
|
+
throw new FileDatabaseError(`[FileDatabase] Version "${version}" not found`);
|
|
871
|
+
}
|
|
872
|
+
await this.setCurrentVersion(version);
|
|
873
|
+
} else {
|
|
874
|
+
await this.setCurrentVersion(versions[versions.length - 1]);
|
|
875
|
+
}
|
|
876
|
+
if (!this.metadata.files.length) {
|
|
877
|
+
this.metadata = await this.figureMetadata(this.currentVersion);
|
|
878
|
+
if (this.metadata.files && this.metadata.files.length > 0) {
|
|
879
|
+
this.currentFileNumber = Math.max(...this.metadata.files.map((f) => f.number || 0));
|
|
880
|
+
} else {
|
|
881
|
+
this.currentFileNumber = 0;
|
|
882
|
+
}
|
|
883
|
+
}
|
|
884
|
+
} else {
|
|
885
|
+
this.currentVersion = null;
|
|
886
|
+
if (this.useMetadata === void 0) {
|
|
887
|
+
const format = await this.detectDataFormat();
|
|
888
|
+
this.useMetadata = format.hasMetadata;
|
|
889
|
+
}
|
|
890
|
+
if (this.useMetadata) {
|
|
891
|
+
const destPath = this.getDestinationPath();
|
|
892
|
+
const metadataPath = path3.join(destPath, "metadata.json");
|
|
893
|
+
if (fs3.existsSync(metadataPath)) {
|
|
894
|
+
try {
|
|
895
|
+
const rawData = await fs3.promises.readFile(metadataPath, "utf8");
|
|
896
|
+
this.metadata = JSON.parse(rawData);
|
|
897
|
+
if (this.metadata.files && this.metadata.files.length > 0) {
|
|
898
|
+
this.currentFileNumber = Math.max(...this.metadata.files.map((f) => f.number || 0));
|
|
899
|
+
} else {
|
|
900
|
+
this.currentFileNumber = 0;
|
|
901
|
+
}
|
|
902
|
+
} catch (e) {
|
|
903
|
+
throw new FileDatabaseError(`Failed to read metadata: ${e.message}`);
|
|
904
|
+
}
|
|
905
|
+
} else {
|
|
906
|
+
throw new FileDatabaseError(
|
|
907
|
+
`[FileDatabase] No metadata found in non-versioned mode. Looked for: ${metadataPath} (table path: ${destPath})`
|
|
908
|
+
);
|
|
909
|
+
}
|
|
910
|
+
} else {
|
|
911
|
+
this.metadata = await this.figureMetadataFromVersionFiles("");
|
|
912
|
+
if (this.metadata.files && this.metadata.files.length > 0) {
|
|
913
|
+
this.currentFileNumber = Math.max(...this.metadata.files.map((f) => f.number || 0));
|
|
914
|
+
} else {
|
|
915
|
+
this.currentFileNumber = 0;
|
|
916
|
+
}
|
|
917
|
+
}
|
|
918
|
+
}
|
|
919
|
+
}
|
|
920
|
+
}
|
|
921
|
+
/**
|
|
922
|
+
* Write data to the file database
|
|
923
|
+
*/
|
|
924
|
+
async write(data, options = {}) {
|
|
925
|
+
if (options.filename) {
|
|
926
|
+
const destPath2 = this.getDestinationPath();
|
|
927
|
+
await ensurePath(destPath2);
|
|
928
|
+
const filePath = path3.join(destPath2, options.filename);
|
|
929
|
+
await this.safeWrite(filePath, data);
|
|
930
|
+
return;
|
|
931
|
+
}
|
|
932
|
+
if (options.forceNewVersion && !this.versioned) {
|
|
933
|
+
throw new FileDatabaseError("Cannot use forceNewVersion in non-versioned mode");
|
|
934
|
+
}
|
|
935
|
+
await this.prepare({ write: true });
|
|
936
|
+
const incomingDataType = detectDataType(data);
|
|
937
|
+
this.metadata.dataType = incomingDataType;
|
|
938
|
+
if (options.forceNewVersion) {
|
|
939
|
+
await this.makeNewVersion();
|
|
940
|
+
this.metadata = this.getDefaultMetadata();
|
|
941
|
+
this.metadata.version = this.currentVersion;
|
|
942
|
+
this.metadata.dataType = incomingDataType;
|
|
943
|
+
this.makeNewFile();
|
|
944
|
+
}
|
|
945
|
+
let targetFileIndex = null;
|
|
946
|
+
const hasCustomMetadata = options.customMetadata && Object.keys(options.customMetadata).length > 0;
|
|
947
|
+
if (hasCustomMetadata) {
|
|
948
|
+
for (let i = 0; i < this.metadata.files.length; i++) {
|
|
949
|
+
const fileEntry = this.metadata.files[i];
|
|
950
|
+
const matches = Object.keys(options.customMetadata).every((key) => {
|
|
951
|
+
return key in fileEntry && fileEntry[key] === options.customMetadata[key];
|
|
952
|
+
});
|
|
953
|
+
if (matches) {
|
|
954
|
+
targetFileIndex = i;
|
|
955
|
+
this.logger.silly?.(`[FileDatabase] Found existing file with matching custom metadata: ${fileEntry.fileName}, metadata: ${JSON.stringify(options.customMetadata)}`);
|
|
956
|
+
break;
|
|
957
|
+
} else {
|
|
958
|
+
this.logger.silly?.(`[FileDatabase] File ${fileEntry.fileName} does not match custom metadata: ${JSON.stringify(options.customMetadata)}`);
|
|
959
|
+
}
|
|
960
|
+
}
|
|
961
|
+
if (targetFileIndex === null) {
|
|
962
|
+
this.logger.silly?.(`[FileDatabase] No existing file found with custom metadata: ${JSON.stringify(options.customMetadata)}, will create new file`);
|
|
963
|
+
}
|
|
964
|
+
} else {
|
|
965
|
+
this.logger.silly?.(`[FileDatabase] No custom metadata provided, will create new file`);
|
|
966
|
+
}
|
|
967
|
+
if (targetFileIndex !== null) {
|
|
968
|
+
const targetFile = this.metadata.files[targetFileIndex];
|
|
969
|
+
this.currentFileNumber = targetFile.number;
|
|
970
|
+
this.lastFileData = null;
|
|
971
|
+
this.currentRecord = 0;
|
|
972
|
+
this.hasReadFirstPage = false;
|
|
973
|
+
}
|
|
974
|
+
const forceNewFile = hasCustomMetadata && targetFileIndex === null;
|
|
975
|
+
let { dataToWrite, dataLeftOver, fileName } = this.figureOutDataAndFileToWrite(data, targetFileIndex, forceNewFile);
|
|
976
|
+
const destPath = this.getDestinationPath(this.currentVersion || void 0);
|
|
977
|
+
await this.safeWrite(path3.join(destPath, fileName), dataToWrite);
|
|
978
|
+
this.updateMetadata(dataToWrite, fileName, options.customMetadata);
|
|
979
|
+
while (dataLeftOver && dataLeftOver.length > 0 && targetFileIndex === null) {
|
|
980
|
+
const writeContext = this.figureOutDataAndFileToWrite(dataLeftOver);
|
|
981
|
+
await this.safeWrite(path3.join(destPath, writeContext.fileName), writeContext.dataToWrite);
|
|
982
|
+
this.updateMetadata(writeContext.dataToWrite, writeContext.fileName, options.customMetadata);
|
|
983
|
+
dataLeftOver = writeContext.dataLeftOver;
|
|
984
|
+
}
|
|
985
|
+
this.calculateVersionSynopsis();
|
|
986
|
+
if (this.useMetadata) {
|
|
987
|
+
await this.saveVersionMetadata(this.metadata);
|
|
988
|
+
}
|
|
989
|
+
}
|
|
990
|
+
/**
|
|
991
|
+
* Read data from the file database
|
|
992
|
+
*/
|
|
993
|
+
async read(options = {}) {
|
|
994
|
+
const { version, nextPage = false, pageSize, filename } = options;
|
|
995
|
+
if (filename) {
|
|
996
|
+
const destPath = this.getDestinationPath(version);
|
|
997
|
+
const filePath = path3.join(destPath, filename);
|
|
998
|
+
try {
|
|
999
|
+
const rawData = await fs3.promises.readFile(filePath, "utf8");
|
|
1000
|
+
return JSON.parse(rawData);
|
|
1001
|
+
} catch (error) {
|
|
1002
|
+
throw new FileDatabaseError(`Failed to read file ${filename}: ${error.message}`);
|
|
1003
|
+
}
|
|
1004
|
+
}
|
|
1005
|
+
await this.prepare({ read: true, version });
|
|
1006
|
+
const isNonPaginatedData = this.metadata.dataType === "text" || this.metadata.dataType === "xml" || this.metadata.dataType === "json-object";
|
|
1007
|
+
if (isNonPaginatedData) {
|
|
1008
|
+
const file = this.metadata.files[0];
|
|
1009
|
+
const filePath = path3.join(this.getDestinationPath(this.currentVersion || void 0), file.fileName);
|
|
1010
|
+
try {
|
|
1011
|
+
const rawData = await fs3.promises.readFile(filePath, "utf8");
|
|
1012
|
+
return deserializeData(rawData, this.metadata.dataType);
|
|
1013
|
+
} catch (error) {
|
|
1014
|
+
throw new FileDatabaseError(`Failed to read file ${file.fileName}: ${error.message}`);
|
|
1015
|
+
}
|
|
1016
|
+
}
|
|
1017
|
+
let effectivePageSize;
|
|
1018
|
+
if (nextPage && this.hasReadFirstPage) {
|
|
1019
|
+
effectivePageSize = pageSize || this.pageSize;
|
|
1020
|
+
this.currentRecord += effectivePageSize;
|
|
1021
|
+
} else if (!nextPage) {
|
|
1022
|
+
effectivePageSize = pageSize !== void 0 ? pageSize : this.metadata.totalRecords;
|
|
1023
|
+
this.currentRecord = 0;
|
|
1024
|
+
} else {
|
|
1025
|
+
effectivePageSize = pageSize || this.pageSize;
|
|
1026
|
+
}
|
|
1027
|
+
if (this.currentRecord >= this.metadata.totalRecords) {
|
|
1028
|
+
return [];
|
|
1029
|
+
}
|
|
1030
|
+
const result = [];
|
|
1031
|
+
let recordsRead = 0;
|
|
1032
|
+
let currentFileIndex = 0;
|
|
1033
|
+
let currentFileOffset = 0;
|
|
1034
|
+
let totalRecords = 0;
|
|
1035
|
+
for (let i = 0; i < this.metadata.files.length; i++) {
|
|
1036
|
+
const file = this.metadata.files[i];
|
|
1037
|
+
if (this.currentRecord < totalRecords + file.recordsCount) {
|
|
1038
|
+
currentFileIndex = i;
|
|
1039
|
+
currentFileOffset = totalRecords;
|
|
1040
|
+
break;
|
|
1041
|
+
}
|
|
1042
|
+
totalRecords += file.recordsCount;
|
|
1043
|
+
}
|
|
1044
|
+
let cumulativeRecords = currentFileOffset;
|
|
1045
|
+
for (let i = currentFileIndex; i < this.metadata.files.length && recordsRead < effectivePageSize; i++) {
|
|
1046
|
+
const file = this.metadata.files[i];
|
|
1047
|
+
const filePath = path3.join(this.getDestinationPath(this.currentVersion || void 0), file.fileName);
|
|
1048
|
+
try {
|
|
1049
|
+
const rawData = await fs3.promises.readFile(filePath, "utf8");
|
|
1050
|
+
const fileData = deserializeData(rawData, this.metadata.dataType);
|
|
1051
|
+
let startIndex = 0;
|
|
1052
|
+
if (i === currentFileIndex) {
|
|
1053
|
+
startIndex = this.currentRecord - cumulativeRecords;
|
|
1054
|
+
}
|
|
1055
|
+
const endIndex = Math.min(startIndex + (effectivePageSize - recordsRead), fileData.length);
|
|
1056
|
+
const recordsFromThisFile = fileData.slice(startIndex, endIndex);
|
|
1057
|
+
result.push(...recordsFromThisFile);
|
|
1058
|
+
recordsRead += recordsFromThisFile.length;
|
|
1059
|
+
cumulativeRecords += file.recordsCount;
|
|
1060
|
+
} catch (error) {
|
|
1061
|
+
throw new FileDatabaseError(`Failed to read file ${file.fileName}: ${error.message}`);
|
|
1062
|
+
}
|
|
1063
|
+
}
|
|
1064
|
+
if (result.length > 0) {
|
|
1065
|
+
if (nextPage || pageSize !== void 0 && pageSize < this.metadata.totalRecords) {
|
|
1066
|
+
this.hasReadFirstPage = true;
|
|
1067
|
+
}
|
|
1068
|
+
}
|
|
1069
|
+
return result;
|
|
1070
|
+
}
|
|
1071
|
+
/**
|
|
1072
|
+
* Set the starting record for pagination (1-based index)
|
|
1073
|
+
*/
|
|
1074
|
+
setStartRecord(startRecord) {
|
|
1075
|
+
this.currentRecord = startRecord - 1;
|
|
1076
|
+
this.hasReadFirstPage = false;
|
|
1077
|
+
}
|
|
1078
|
+
/**
|
|
1079
|
+
* Reset read pagination state
|
|
1080
|
+
*/
|
|
1081
|
+
resetPagination() {
|
|
1082
|
+
this.currentRecord = 0;
|
|
1083
|
+
this.hasReadFirstPage = false;
|
|
1084
|
+
}
|
|
1085
|
+
/**
|
|
1086
|
+
* List filenames in the table directory.
|
|
1087
|
+
* For catalog/key-value usage (files written with { filename }).
|
|
1088
|
+
* Returns data file names (.json, .txt, .xml) excluding metadata.json.
|
|
1089
|
+
*/
|
|
1090
|
+
async listFilenames() {
|
|
1091
|
+
const destPath = this.versioned && this.currentVersion ? path3.join(this.getDestinationPath(), this.currentVersion) : this.getDestinationPath();
|
|
1092
|
+
try {
|
|
1093
|
+
const entries = await fs3.promises.readdir(destPath, { withFileTypes: true });
|
|
1094
|
+
return entries.filter((e) => e.isFile() && e.name !== "metadata.json" && /\.(json|txt|xml)$/i.test(e.name)).map((e) => e.name);
|
|
1095
|
+
} catch (err) {
|
|
1096
|
+
if (err?.code === "ENOENT") return [];
|
|
1097
|
+
throw new FileDatabaseError(`Failed to list files: ${err.message}`);
|
|
1098
|
+
}
|
|
1099
|
+
}
|
|
1100
|
+
/**
|
|
1101
|
+
* Remove a file from the table directory (catalog mode).
|
|
1102
|
+
* Use with listFilenames() to manage individual files.
|
|
1103
|
+
*/
|
|
1104
|
+
async removeFile(filename) {
|
|
1105
|
+
const destPath = this.versioned && this.currentVersion ? path3.join(this.getDestinationPath(), this.currentVersion) : this.getDestinationPath();
|
|
1106
|
+
const filePath = path3.join(destPath, filename);
|
|
1107
|
+
try {
|
|
1108
|
+
await fs3.promises.unlink(filePath);
|
|
1109
|
+
} catch (err) {
|
|
1110
|
+
if (err?.code === "ENOENT") return;
|
|
1111
|
+
throw new FileDatabaseError(`Failed to remove file ${filename}: ${err.message}`);
|
|
1112
|
+
}
|
|
1113
|
+
}
|
|
1114
|
+
/**
|
|
1115
|
+
* Remove a file and its metadata entry (non-versioned mode with useMetadata).
|
|
1116
|
+
* Use with findData() to get fileName, then call removeFileEntry to delete.
|
|
1117
|
+
*/
|
|
1118
|
+
async removeFileEntry(filename) {
|
|
1119
|
+
if (this.versioned) {
|
|
1120
|
+
throw new FileDatabaseError("removeFileEntry is only supported in non-versioned mode");
|
|
1121
|
+
}
|
|
1122
|
+
await this.prepare({ read: true });
|
|
1123
|
+
const idx = this.metadata.files.findIndex((f) => f.fileName === filename);
|
|
1124
|
+
if (idx === -1) {
|
|
1125
|
+
throw new FileDatabaseError(`File entry ${filename} not found in metadata`);
|
|
1126
|
+
}
|
|
1127
|
+
const entry = this.metadata.files[idx];
|
|
1128
|
+
const recordsCount = entry.recordsCount || 0;
|
|
1129
|
+
this.metadata.files.splice(idx, 1);
|
|
1130
|
+
this.metadata.totalRecords = Math.max(0, (this.metadata.totalRecords || 0) - recordsCount);
|
|
1131
|
+
const destPath = this.getDestinationPath();
|
|
1132
|
+
const filePath = path3.join(destPath, filename);
|
|
1133
|
+
try {
|
|
1134
|
+
await fs3.promises.unlink(filePath);
|
|
1135
|
+
} catch (err) {
|
|
1136
|
+
if (err?.code === "ENOENT") {
|
|
1137
|
+
this.logger.warn?.(`[FileDatabase] File ${filename} already missing on disk`);
|
|
1138
|
+
} else {
|
|
1139
|
+
throw new FileDatabaseError(`Failed to remove file ${filename}: ${err.message}`);
|
|
1140
|
+
}
|
|
1141
|
+
}
|
|
1142
|
+
if (this.useMetadata) {
|
|
1143
|
+
await this.saveVersionMetadata(this.metadata);
|
|
1144
|
+
}
|
|
1145
|
+
}
|
|
1146
|
+
/**
|
|
1147
|
+
* Set file-level synopsis calculation function
|
|
1148
|
+
*/
|
|
1149
|
+
setFileSynopsisFunction(fn) {
|
|
1150
|
+
this.fileSynopsisFunction = fn;
|
|
1151
|
+
}
|
|
1152
|
+
/**
|
|
1153
|
+
* Set version-level synopsis calculation function
|
|
1154
|
+
*/
|
|
1155
|
+
setVersionSynopsisFunction(fn) {
|
|
1156
|
+
this.versionSynopsisFunction = fn;
|
|
1157
|
+
}
|
|
1158
|
+
/**
|
|
1159
|
+
* Get current version name
|
|
1160
|
+
*/
|
|
1161
|
+
getCurrentVersion() {
|
|
1162
|
+
return this.currentVersion;
|
|
1163
|
+
}
|
|
1164
|
+
/**
|
|
1165
|
+
* Get current metadata
|
|
1166
|
+
*/
|
|
1167
|
+
getMetadata() {
|
|
1168
|
+
return { ...this.metadata };
|
|
1169
|
+
}
|
|
1170
|
+
/**
|
|
1171
|
+
* Find data by custom metadata fields
|
|
1172
|
+
* Searches through all versions and files to find entries matching the search criteria
|
|
1173
|
+
*
|
|
1174
|
+
* @param searchCriteria - Object with field names and values to search for (e.g., { ListingKey: "123", id: "456" })
|
|
1175
|
+
* @returns Array of found entries with their file paths and metadata
|
|
1176
|
+
*/
|
|
1177
|
+
async findData(searchCriteria) {
|
|
1178
|
+
const results = [];
|
|
1179
|
+
if (!this.versioned) {
|
|
1180
|
+
await this.prepare({ read: true });
|
|
1181
|
+
const metadata = this.getMetadata();
|
|
1182
|
+
for (const fileEntry of metadata.files) {
|
|
1183
|
+
const matches = Object.keys(searchCriteria).every((key) => {
|
|
1184
|
+
return fileEntry[key] === searchCriteria[key];
|
|
1185
|
+
});
|
|
1186
|
+
if (matches) {
|
|
1187
|
+
const destPath = this.getDestinationPath();
|
|
1188
|
+
const filePath = path3.join(destPath, fileEntry.fileName);
|
|
1189
|
+
const fileData = await fs3.promises.readFile(filePath, "utf8");
|
|
1190
|
+
const data = deserializeData(fileData, metadata.dataType || "json-object");
|
|
1191
|
+
results.push({
|
|
1192
|
+
filePath,
|
|
1193
|
+
fileName: fileEntry.fileName,
|
|
1194
|
+
version: null,
|
|
1195
|
+
metadata: fileEntry,
|
|
1196
|
+
data
|
|
1197
|
+
});
|
|
1198
|
+
}
|
|
1199
|
+
}
|
|
1200
|
+
} else {
|
|
1201
|
+
const versions = await this.getVersions();
|
|
1202
|
+
for (const version of versions) {
|
|
1203
|
+
await this.prepare({ read: true, version });
|
|
1204
|
+
const metadata = this.getMetadata();
|
|
1205
|
+
for (const fileEntry of metadata.files) {
|
|
1206
|
+
const matches = Object.keys(searchCriteria).every((key) => {
|
|
1207
|
+
return fileEntry[key] === searchCriteria[key];
|
|
1208
|
+
});
|
|
1209
|
+
if (matches) {
|
|
1210
|
+
const destPath = this.getDestinationPath(version);
|
|
1211
|
+
const filePath = path3.join(destPath, fileEntry.fileName);
|
|
1212
|
+
const fileData = await fs3.promises.readFile(filePath, "utf8");
|
|
1213
|
+
const data = deserializeData(fileData, metadata.dataType || "json-object");
|
|
1214
|
+
results.push({
|
|
1215
|
+
filePath,
|
|
1216
|
+
fileName: fileEntry.fileName,
|
|
1217
|
+
version,
|
|
1218
|
+
metadata: fileEntry,
|
|
1219
|
+
data
|
|
1220
|
+
});
|
|
1221
|
+
}
|
|
1222
|
+
}
|
|
1223
|
+
}
|
|
1224
|
+
}
|
|
1225
|
+
return results;
|
|
1226
|
+
}
|
|
1227
|
+
};
|
|
1228
|
+
|
|
1229
|
+
// src/mock-server/mock-storage.ts
|
|
1230
|
+
function stableStringify(obj) {
|
|
1231
|
+
if (obj === null) return "null";
|
|
1232
|
+
if (obj === void 0) return "undefined";
|
|
1233
|
+
if (typeof obj !== "object") return JSON.stringify(obj);
|
|
1234
|
+
if (Array.isArray(obj)) return "[" + obj.map(stableStringify).join(",") + "]";
|
|
1235
|
+
const keys = Object.keys(obj).sort();
|
|
1236
|
+
return "{" + keys.map((k) => JSON.stringify(k) + ":" + stableStringify(obj[k])).join(",") + "}";
|
|
1237
|
+
}
|
|
1238
|
+
function normalizeQuery(query) {
|
|
1239
|
+
if (!query) return "";
|
|
1240
|
+
const params = new URLSearchParams(query);
|
|
1241
|
+
const sorted = [...params.entries()].sort((a, b) => a[0].localeCompare(b[0]));
|
|
1242
|
+
return new URLSearchParams(sorted).toString();
|
|
1243
|
+
}
|
|
1244
|
+
function buildCriteria(method, host, pathname, query, requestData) {
|
|
1245
|
+
return {
|
|
1246
|
+
method: method.toUpperCase(),
|
|
1247
|
+
host,
|
|
1248
|
+
pathname: pathname || "/",
|
|
1249
|
+
query: normalizeQuery(query),
|
|
1250
|
+
requestBody: requestData != null ? stableStringify(requestData) : ""
|
|
1251
|
+
};
|
|
1252
|
+
}
|
|
1253
|
+
var MockStorage = class {
|
|
1254
|
+
fileDb;
|
|
1255
|
+
logger;
|
|
1256
|
+
constructor(config) {
|
|
1257
|
+
this.logger = config.logger ?? console;
|
|
1258
|
+
this.fileDb = new FileDatabase({
|
|
1259
|
+
basePath: config.basePath,
|
|
1260
|
+
namespace: "mocks",
|
|
1261
|
+
tableName: "responses",
|
|
1262
|
+
versioned: false,
|
|
1263
|
+
useMetadata: true,
|
|
1264
|
+
logger: this.logger
|
|
1265
|
+
});
|
|
1266
|
+
}
|
|
1267
|
+
/**
|
|
1268
|
+
* Store a mock response.
|
|
1269
|
+
*/
|
|
1270
|
+
async store(method, requestUrl, requestData, responseData) {
|
|
1271
|
+
const url = new URL(requestUrl);
|
|
1272
|
+
const criteria = buildCriteria(method, url.host, url.pathname, url.search.slice(1), requestData);
|
|
1273
|
+
await this.fileDb.write(responseData, { customMetadata: criteria });
|
|
1274
|
+
}
|
|
1275
|
+
/**
|
|
1276
|
+
* Find a mock response by request criteria. Returns null if not found.
|
|
1277
|
+
*/
|
|
1278
|
+
async find(method, host, pathname, query, requestData) {
|
|
1279
|
+
const criteria = buildCriteria(method, host, pathname, query, requestData);
|
|
1280
|
+
try {
|
|
1281
|
+
const results = await this.fileDb.findData(criteria);
|
|
1282
|
+
if (results.length === 0) return null;
|
|
1283
|
+
return results[0].data;
|
|
1284
|
+
} catch (err) {
|
|
1285
|
+
if (err instanceof FileDatabaseError && /No metadata found/.test(err.message)) {
|
|
1286
|
+
return null;
|
|
1287
|
+
}
|
|
1288
|
+
throw err;
|
|
1289
|
+
}
|
|
1290
|
+
}
|
|
1291
|
+
/**
|
|
1292
|
+
* List all stored mock keys (opaque identifiers for remove)
|
|
1293
|
+
*/
|
|
1294
|
+
async listKeys() {
|
|
1295
|
+
const files = await this.fileDb.listFilenames();
|
|
1296
|
+
return files.sort();
|
|
1297
|
+
}
|
|
1298
|
+
/**
|
|
1299
|
+
* Remove a mock by key (from listKeys)
|
|
1300
|
+
*/
|
|
1301
|
+
async remove(fileName) {
|
|
1302
|
+
const name = fileName.endsWith(".json") ? fileName : `${fileName}.json`;
|
|
1303
|
+
try {
|
|
1304
|
+
await this.fileDb.removeFileEntry(name);
|
|
1305
|
+
return true;
|
|
1306
|
+
} catch {
|
|
1307
|
+
return false;
|
|
1308
|
+
}
|
|
1309
|
+
}
|
|
1310
|
+
/**
|
|
1311
|
+
* Remove a mock by request criteria (method, host, pathname, query, requestData)
|
|
1312
|
+
*/
|
|
1313
|
+
async removeByCriteria(method, host, pathname, query, requestData) {
|
|
1314
|
+
const criteria = buildCriteria(method, host, pathname, query, requestData);
|
|
1315
|
+
try {
|
|
1316
|
+
const results = await this.fileDb.findData(criteria);
|
|
1317
|
+
if (results.length === 0) return false;
|
|
1318
|
+
await this.fileDb.removeFileEntry(results[0].fileName);
|
|
1319
|
+
return true;
|
|
1320
|
+
} catch {
|
|
1321
|
+
return false;
|
|
1322
|
+
}
|
|
1323
|
+
}
|
|
1324
|
+
};
|
|
1325
|
+
|
|
116
1326
|
// src/http-client2/index.ts
|
|
117
1327
|
function buildUrl(baseURL, url, params) {
|
|
118
1328
|
let full = url;
|
|
@@ -165,14 +1375,50 @@ function mapStatusToCustom(httpStatus) {
|
|
|
165
1375
|
if (httpStatus >= 500) return "serverError";
|
|
166
1376
|
return "unknown";
|
|
167
1377
|
}
|
|
1378
|
+
function formatForDisplay(value, maxKeys, maxArrayItems, maxChars) {
|
|
1379
|
+
if (value === null) return "null";
|
|
1380
|
+
if (value === void 0) return "undefined";
|
|
1381
|
+
if (typeof value === "boolean" || typeof value === "number") return String(value);
|
|
1382
|
+
if (typeof value === "string") {
|
|
1383
|
+
if (value.length <= maxChars) return JSON.stringify(value);
|
|
1384
|
+
return JSON.stringify(value.slice(0, maxChars) + `... (+${value.length - maxChars} chars)`);
|
|
1385
|
+
}
|
|
1386
|
+
if (Array.isArray(value)) {
|
|
1387
|
+
const head = value.slice(0, maxArrayItems);
|
|
1388
|
+
const rest = value.length - maxArrayItems;
|
|
1389
|
+
const items = head.map((v) => formatForDisplay(v, maxKeys, maxArrayItems, maxChars));
|
|
1390
|
+
if (rest <= 0) return `[${items.join(", ")}]`;
|
|
1391
|
+
return `[${items.join(", ")}, ... +${rest} more]`;
|
|
1392
|
+
}
|
|
1393
|
+
if (typeof value === "object") {
|
|
1394
|
+
const keys = Object.keys(value);
|
|
1395
|
+
const shown = keys.slice(0, maxKeys);
|
|
1396
|
+
const rest = keys.length - maxKeys;
|
|
1397
|
+
const pairs = shown.map((k) => `${JSON.stringify(k)}: ${formatForDisplay(value[k], maxKeys, maxArrayItems, maxChars)}`);
|
|
1398
|
+
if (rest <= 0) return `{${pairs.join(", ")}}`;
|
|
1399
|
+
return `{${pairs.join(", ")}, ... +${rest} more keys}`;
|
|
1400
|
+
}
|
|
1401
|
+
return String(value);
|
|
1402
|
+
}
|
|
168
1403
|
var HttpClient = class _HttpClient {
|
|
169
1404
|
context;
|
|
170
1405
|
config;
|
|
171
1406
|
logger;
|
|
1407
|
+
mockStorage = null;
|
|
172
1408
|
constructor(context, options = {}) {
|
|
173
1409
|
this.context = context;
|
|
174
1410
|
this.config = options;
|
|
175
1411
|
this.logger = options.logger ?? context?.logger ?? console;
|
|
1412
|
+
if (options.saveMock || options.useMock) {
|
|
1413
|
+
const mocksPath = options.mocksPath;
|
|
1414
|
+
if (!mocksPath) {
|
|
1415
|
+
throw new ParamError("[http-client2] mocksPath is required when saveMock or useMock is set");
|
|
1416
|
+
}
|
|
1417
|
+
const absoluteMocksPath = path4.resolve(mocksPath);
|
|
1418
|
+
this.mockStorage = new MockStorage({ basePath: absoluteMocksPath, logger: this.logger });
|
|
1419
|
+
const mode = [options.saveMock && "saveMock", options.useMock && "useMock"].filter(Boolean).join(", ");
|
|
1420
|
+
this.logger.debug?.(`[HttpClient] mocks enabled (${mode}), mocksPath=${absoluteMocksPath}`);
|
|
1421
|
+
}
|
|
176
1422
|
}
|
|
177
1423
|
/**
|
|
178
1424
|
* Static init - discovers params via context.params.getAllForModule("http-client2", defs). Whatever is in options goes.
|
|
@@ -185,27 +1431,85 @@ var HttpClient = class _HttpClient {
|
|
|
185
1431
|
maxRetryDelay: "number default 30000",
|
|
186
1432
|
retryJitter: "number default 0.1",
|
|
187
1433
|
userAgent: "string default HttpClient/v1.0",
|
|
188
|
-
baseURL: "string"
|
|
1434
|
+
baseURL: "string",
|
|
1435
|
+
saveMock: "boolean default false",
|
|
1436
|
+
useMock: "boolean default false",
|
|
1437
|
+
mocksPath: "string",
|
|
1438
|
+
useTestServer: "string",
|
|
1439
|
+
showRequest: "boolean default false",
|
|
1440
|
+
showResponse: "boolean default false",
|
|
1441
|
+
showRequestHeaders: "boolean default false",
|
|
1442
|
+
showResponseHeaders: "boolean default false",
|
|
1443
|
+
showMaxKeys: "number default 20",
|
|
1444
|
+
showMaxArrayItems: "number default 5",
|
|
1445
|
+
showMaxChars: "number default 300"
|
|
189
1446
|
};
|
|
190
1447
|
const discovered = context?.params?.getAllForModule?.(defs) ?? {};
|
|
191
1448
|
const merged = { ...discovered, ...options };
|
|
1449
|
+
if ((merged.saveMock || merged.useMock) && !merged.mocksPath) {
|
|
1450
|
+
throw new ParamError("[http-client2] mocksPath is required when saveMock or useMock is set");
|
|
1451
|
+
}
|
|
1452
|
+
if ((merged.saveMock || merged.useMock) && merged.useTestServer) {
|
|
1453
|
+
throw new ParamError("[http-client2] saveMock/useMock cannot be used with useTestServer");
|
|
1454
|
+
}
|
|
192
1455
|
return new _HttpClient(context, merged);
|
|
193
1456
|
}
|
|
194
1457
|
async request(method, url, options = {}) {
|
|
195
1458
|
const startTime = Date.now();
|
|
196
1459
|
const timeout = options.timeout ?? this.config.timeout;
|
|
197
|
-
const retryCount = options.retryCount ?? this.config.retryCount;
|
|
198
|
-
const retryDelay = options.retryDelay ?? this.config.retryDelay;
|
|
199
|
-
|
|
1460
|
+
const retryCount = options.retryCount ?? this.config.retryCount ?? 3;
|
|
1461
|
+
const retryDelay = options.retryDelay ?? this.config.retryDelay ?? 1e3;
|
|
1462
|
+
let fullUrl = buildUrl(this.config.baseURL, url, options.params);
|
|
1463
|
+
if (this.config.useMock && this.mockStorage) {
|
|
1464
|
+
try {
|
|
1465
|
+
const urlObj = new URL(fullUrl);
|
|
1466
|
+
const mock = await this.mockStorage.find(
|
|
1467
|
+
method,
|
|
1468
|
+
urlObj.host,
|
|
1469
|
+
urlObj.pathname,
|
|
1470
|
+
urlObj.search.slice(1),
|
|
1471
|
+
options.data
|
|
1472
|
+
);
|
|
1473
|
+
if (mock) {
|
|
1474
|
+
const duration = Date.now() - startTime;
|
|
1475
|
+
if (this.config.showResponse) {
|
|
1476
|
+
this.logRequestResponse(method, fullUrl, options, null, mock, duration, true);
|
|
1477
|
+
}
|
|
1478
|
+
return {
|
|
1479
|
+
status: mapStatusToCustom(mock.status),
|
|
1480
|
+
code: mock.status,
|
|
1481
|
+
headers: mock.headers || {},
|
|
1482
|
+
data: mock.data,
|
|
1483
|
+
duration,
|
|
1484
|
+
retryCount: 0,
|
|
1485
|
+
finalUrl: fullUrl
|
|
1486
|
+
};
|
|
1487
|
+
}
|
|
1488
|
+
throw new HttpClientError(`Mock not found for ${method} ${fullUrl}`);
|
|
1489
|
+
} catch (e) {
|
|
1490
|
+
if (e instanceof HttpClientError) throw e;
|
|
1491
|
+
this.logger.warn?.("[HttpClient] Mock lookup failed:", e);
|
|
1492
|
+
throw new HttpClientError(`Mock not found for ${method} ${fullUrl} (lookup failed: ${e.message})`);
|
|
1493
|
+
}
|
|
1494
|
+
}
|
|
1495
|
+
let fetchUrl = fullUrl;
|
|
200
1496
|
const headers = {
|
|
201
|
-
"User-Agent": options.userAgent ?? this.config.userAgent,
|
|
1497
|
+
"User-Agent": options.userAgent ?? this.config.userAgent ?? "HttpClient/v1.0",
|
|
202
1498
|
...options.headers
|
|
203
1499
|
};
|
|
1500
|
+
if (this.config.useTestServer) {
|
|
1501
|
+
const urlObj = new URL(fullUrl);
|
|
1502
|
+
fetchUrl = this.config.useTestServer.replace(/\/$/, "") + urlObj.pathname + urlObj.search;
|
|
1503
|
+
headers["XAXIOSOrigin"] = fullUrl;
|
|
1504
|
+
}
|
|
204
1505
|
let body;
|
|
205
1506
|
if (options.data != null && ["POST", "PUT", "PATCH"].includes(method)) {
|
|
206
1507
|
body = typeof options.data === "string" ? options.data : JSON.stringify(options.data);
|
|
207
1508
|
if (!headers["content-type"]) headers["Content-Type"] = "application/json";
|
|
208
1509
|
}
|
|
1510
|
+
if (this.config.showRequest) {
|
|
1511
|
+
this.logRequestResponse(method, fullUrl, options, headers, null, 0, false);
|
|
1512
|
+
}
|
|
209
1513
|
let lastError = null;
|
|
210
1514
|
for (let attempt = 1; attempt <= retryCount + 1; attempt++) {
|
|
211
1515
|
let abortedByTimeout = false;
|
|
@@ -218,7 +1522,7 @@ var HttpClient = class _HttpClient {
|
|
|
218
1522
|
abortedByTimeout = true;
|
|
219
1523
|
controller.abort();
|
|
220
1524
|
}, timeout);
|
|
221
|
-
const response = await fetch(
|
|
1525
|
+
const response = await fetch(fetchUrl, {
|
|
222
1526
|
method,
|
|
223
1527
|
headers,
|
|
224
1528
|
body: method !== "GET" && method !== "HEAD" ? body : void 0,
|
|
@@ -233,7 +1537,7 @@ var HttpClient = class _HttpClient {
|
|
|
233
1537
|
if (options.debug) {
|
|
234
1538
|
this.logger.debug?.(`[HttpClient] ${method} ${fullUrl} \u2192 ${response.status} success (${duration}ms)`);
|
|
235
1539
|
}
|
|
236
|
-
|
|
1540
|
+
const result = {
|
|
237
1541
|
status: mapStatusToCustom(response.status),
|
|
238
1542
|
code: response.status,
|
|
239
1543
|
headers: resHeaders,
|
|
@@ -242,14 +1546,38 @@ var HttpClient = class _HttpClient {
|
|
|
242
1546
|
retryCount: attempt - 1,
|
|
243
1547
|
finalUrl: response.url || fullUrl
|
|
244
1548
|
};
|
|
1549
|
+
if (this.config.saveMock && this.mockStorage) {
|
|
1550
|
+
try {
|
|
1551
|
+
const responseData = {
|
|
1552
|
+
status: response.status,
|
|
1553
|
+
headers: resHeaders,
|
|
1554
|
+
data: data2
|
|
1555
|
+
};
|
|
1556
|
+
await this.mockStorage.store(
|
|
1557
|
+
method,
|
|
1558
|
+
fullUrl,
|
|
1559
|
+
options.data,
|
|
1560
|
+
responseData
|
|
1561
|
+
);
|
|
1562
|
+
} catch (e) {
|
|
1563
|
+
this.logger.warn?.("[HttpClient] Failed to save mock:", e);
|
|
1564
|
+
}
|
|
1565
|
+
}
|
|
1566
|
+
if (this.config.showResponse) {
|
|
1567
|
+
this.logRequestResponse(method, fullUrl, options, headers, { status: response.status, headers: resHeaders, data: data2 }, duration, false);
|
|
1568
|
+
}
|
|
1569
|
+
return result;
|
|
245
1570
|
}
|
|
246
1571
|
const data = await parseBody(response);
|
|
247
1572
|
const classification2 = classifyError({ response: { status: response.status, headers: resHeaders, data } });
|
|
1573
|
+
if (this.config.showResponse) {
|
|
1574
|
+
this.logRequestResponse(method, fullUrl, options, headers, { status: response.status, headers: resHeaders, data }, duration, false);
|
|
1575
|
+
}
|
|
248
1576
|
if (options.debug) {
|
|
249
1577
|
this.logger.debug?.(`[HttpClient] ${method} ${fullUrl} \u2192 ${response.status} ${classification2.status} (${duration}ms)`);
|
|
250
1578
|
}
|
|
251
1579
|
if (attempt <= retryCount && shouldRetryError(classification2)) {
|
|
252
|
-
const delay = calculateRetryDelay(attempt, retryDelay, this.config.maxRetryDelay, this.config.retryJitter);
|
|
1580
|
+
const delay = calculateRetryDelay(attempt, retryDelay, this.config.maxRetryDelay ?? 3e4, this.config.retryJitter ?? 0.1);
|
|
253
1581
|
this.logger.warn?.(`[HttpClient] ${method} ${fullUrl} failed (${classification2.type}). Retrying in ${delay}ms...`);
|
|
254
1582
|
if (options.debug) this.logger.debug?.(`[HttpClient] Waiting ${delay}ms before retry ${attempt + 1}`);
|
|
255
1583
|
await sleep(delay);
|
|
@@ -279,7 +1607,7 @@ var HttpClient = class _HttpClient {
|
|
|
279
1607
|
this.logger.error?.(`[HttpClient] ${method} ${fullUrl} failed (${classification2.type}): ${getErrorDescription(classification2.type)}`);
|
|
280
1608
|
}
|
|
281
1609
|
if (attempt <= retryCount && shouldRetryError(classification2)) {
|
|
282
|
-
const delay = calculateRetryDelay(attempt, retryDelay, this.config.maxRetryDelay, this.config.retryJitter);
|
|
1610
|
+
const delay = calculateRetryDelay(attempt, retryDelay, this.config.maxRetryDelay ?? 3e4, this.config.retryJitter ?? 0.1);
|
|
283
1611
|
if (options.debug) this.logger.debug?.(`[HttpClient] Waiting ${delay}ms before retry ${attempt + 1}`);
|
|
284
1612
|
await sleep(delay);
|
|
285
1613
|
continue;
|
|
@@ -308,6 +1636,28 @@ var HttpClient = class _HttpClient {
|
|
|
308
1636
|
finalUrl: fullUrl
|
|
309
1637
|
};
|
|
310
1638
|
}
|
|
1639
|
+
logRequestResponse(method, url, options, reqHeaders, res, durationMs, fromMock) {
|
|
1640
|
+
const maxKeys = this.config.showMaxKeys ?? 20;
|
|
1641
|
+
const maxArray = this.config.showMaxArrayItems ?? 5;
|
|
1642
|
+
const maxChars = this.config.showMaxChars ?? 300;
|
|
1643
|
+
const log = this.logger.info ?? this.logger.log ?? console.log;
|
|
1644
|
+
log(`[HttpClient] ${method} ${url}`);
|
|
1645
|
+
if (this.config.showRequestHeaders && reqHeaders && Object.keys(reqHeaders).length > 0) {
|
|
1646
|
+
log(` Request headers: ${formatForDisplay(reqHeaders, maxKeys, maxArray, maxChars)}`);
|
|
1647
|
+
}
|
|
1648
|
+
if (options.data != null) {
|
|
1649
|
+
log(` Request body: ${formatForDisplay(options.data, maxKeys, maxArray, maxChars)}`);
|
|
1650
|
+
}
|
|
1651
|
+
if (res) {
|
|
1652
|
+
log(` \u2192 ${res.status}${fromMock ? " (from mock)" : ""} ${durationMs}ms`);
|
|
1653
|
+
if (this.config.showResponseHeaders && res.headers && Object.keys(res.headers).length > 0) {
|
|
1654
|
+
log(` Response headers: ${formatForDisplay(res.headers, maxKeys, maxArray, maxChars)}`);
|
|
1655
|
+
}
|
|
1656
|
+
if (res.data != null) {
|
|
1657
|
+
log(` Response body: ${formatForDisplay(res.data, maxKeys, maxArray, maxChars)}`);
|
|
1658
|
+
}
|
|
1659
|
+
}
|
|
1660
|
+
}
|
|
311
1661
|
async get(url, options = {}) {
|
|
312
1662
|
return this.request("GET", url, options);
|
|
313
1663
|
}
|