@trebired/logger 1.1.3 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/README.md +152 -3
- package/dist/browser/console_transport.d.ts +5 -0
- package/dist/browser/console_transport.d.ts.map +1 -0
- package/dist/browser/console_transport.js +51 -0
- package/dist/browser/console_transport.js.map +1 -0
- package/dist/browser/create_log.d.ts +4 -0
- package/dist/browser/create_log.d.ts.map +1 -0
- package/dist/browser/create_log.js +234 -0
- package/dist/browser/create_log.js.map +1 -0
- package/dist/browser/index.d.ts +4 -0
- package/dist/browser/index.d.ts.map +1 -0
- package/dist/browser/index.js +3 -0
- package/dist/browser/index.js.map +1 -0
- package/dist/browser/react.d.ts +30 -0
- package/dist/browser/react.d.ts.map +1 -0
- package/dist/browser/react.js +44 -0
- package/dist/browser/react.js.map +1 -0
- package/dist/core/create_log.d.ts.map +1 -1
- package/dist/core/create_log.js +121 -145
- package/dist/core/create_log.js.map +1 -1
- package/dist/core/shared.d.ts +31 -0
- package/dist/core/shared.d.ts.map +1 -0
- package/dist/core/shared.js +144 -0
- package/dist/core/shared.js.map +1 -0
- package/dist/format/console.d.ts +2 -2
- package/dist/format/console.d.ts.map +1 -1
- package/dist/format/console.js +40 -21
- package/dist/format/console.js.map +1 -1
- package/dist/format/options.d.ts +4 -0
- package/dist/format/options.d.ts.map +1 -0
- package/dist/format/options.js +19 -0
- package/dist/format/options.js.map +1 -0
- package/dist/index.d.ts +4 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -1
- package/dist/storage/backend/index.d.ts +6 -0
- package/dist/storage/backend/index.d.ts.map +1 -0
- package/dist/storage/backend/index.js +19 -0
- package/dist/storage/backend/index.js.map +1 -0
- package/dist/storage/backend/js.d.ts +4 -0
- package/dist/storage/backend/js.d.ts.map +1 -0
- package/dist/storage/backend/js.js +179 -0
- package/dist/storage/backend/js.js.map +1 -0
- package/dist/storage/backend/native.d.ts +6 -0
- package/dist/storage/backend/native.d.ts.map +1 -0
- package/dist/storage/backend/native.js +89 -0
- package/dist/storage/backend/native.js.map +1 -0
- package/dist/storage/backend/types.d.ts +57 -0
- package/dist/storage/backend/types.d.ts.map +1 -0
- package/dist/storage/backend/types.js +2 -0
- package/dist/storage/backend/types.js.map +1 -0
- package/dist/storage/export.d.ts +5 -0
- package/dist/storage/export.d.ts.map +1 -0
- package/dist/storage/export.js +124 -0
- package/dist/storage/export.js.map +1 -0
- package/dist/storage/names.d.ts +6 -2
- package/dist/storage/names.d.ts.map +1 -1
- package/dist/storage/names.js +66 -31
- package/dist/storage/names.js.map +1 -1
- package/dist/storage/partitions/delete.d.ts +5 -0
- package/dist/storage/partitions/delete.d.ts.map +1 -0
- package/dist/storage/partitions/delete.js +121 -0
- package/dist/storage/partitions/delete.js.map +1 -0
- package/dist/storage/partitions/files.d.ts +12 -0
- package/dist/storage/partitions/files.d.ts.map +1 -0
- package/dist/storage/partitions/files.js +142 -0
- package/dist/storage/partitions/files.js.map +1 -0
- package/dist/storage/partitions/internal.d.ts +47 -0
- package/dist/storage/partitions/internal.d.ts.map +1 -0
- package/dist/storage/partitions/internal.js +35 -0
- package/dist/storage/partitions/internal.js.map +1 -0
- package/dist/storage/partitions/markers.d.ts +8 -0
- package/dist/storage/partitions/markers.d.ts.map +1 -0
- package/dist/storage/partitions/markers.js +88 -0
- package/dist/storage/partitions/markers.js.map +1 -0
- package/dist/storage/partitions/public.d.ts +11 -0
- package/dist/storage/partitions/public.d.ts.map +1 -0
- package/dist/storage/partitions/public.js +79 -0
- package/dist/storage/partitions/public.js.map +1 -0
- package/dist/storage/partitions/records.d.ts +9 -0
- package/dist/storage/partitions/records.d.ts.map +1 -0
- package/dist/storage/partitions/records.js +73 -0
- package/dist/storage/partitions/records.js.map +1 -0
- package/dist/storage/partitions/transforms.d.ts +6 -0
- package/dist/storage/partitions/transforms.d.ts.map +1 -0
- package/dist/storage/partitions/transforms.js +59 -0
- package/dist/storage/partitions/transforms.js.map +1 -0
- package/dist/storage/partitions.d.ts +4 -0
- package/dist/storage/partitions.d.ts.map +1 -0
- package/dist/storage/partitions.js +4 -0
- package/dist/storage/partitions.js.map +1 -0
- package/dist/storage/query.d.ts +3 -3
- package/dist/storage/query.d.ts.map +1 -1
- package/dist/storage/query.js +29 -31
- package/dist/storage/query.js.map +1 -1
- package/dist/storage/retention.d.ts.map +1 -1
- package/dist/storage/retention.js +2 -5
- package/dist/storage/retention.js.map +1 -1
- package/dist/storage/walk.d.ts.map +1 -1
- package/dist/storage/walk.js +3 -20
- package/dist/storage/walk.js.map +1 -1
- package/dist/storage/write.d.ts +1 -2
- package/dist/storage/write.d.ts.map +1 -1
- package/dist/storage/write.js +10 -12
- package/dist/storage/write.js.map +1 -1
- package/dist/stream/index.d.ts +11 -3
- package/dist/stream/index.d.ts.map +1 -1
- package/dist/stream/index.js +40 -4
- package/dist/stream/index.js.map +1 -1
- package/dist/types/browser.d.ts +54 -0
- package/dist/types/browser.d.ts.map +1 -0
- package/dist/types/browser.js +2 -0
- package/dist/types/browser.js.map +1 -0
- package/dist/types/common.d.ts +73 -0
- package/dist/types/common.d.ts.map +1 -0
- package/dist/types/common.js +2 -0
- package/dist/types/common.js.map +1 -0
- package/dist/types/export.d.ts +53 -0
- package/dist/types/export.d.ts.map +1 -0
- package/dist/types/export.js +2 -0
- package/dist/types/export.js.map +1 -0
- package/dist/types/index.d.ts +7 -0
- package/dist/types/index.d.ts.map +1 -0
- package/dist/types/index.js +2 -0
- package/dist/types/index.js.map +1 -0
- package/dist/types/partitions.d.ts +92 -0
- package/dist/types/partitions.d.ts.map +1 -0
- package/dist/types/partitions.js +2 -0
- package/dist/types/partitions.js.map +1 -0
- package/dist/types/query.d.ts +49 -0
- package/dist/types/query.d.ts.map +1 -0
- package/dist/types/query.js +2 -0
- package/dist/types/query.js.map +1 -0
- package/dist/types/server.d.ts +79 -0
- package/dist/types/server.d.ts.map +1 -0
- package/dist/types/server.js +2 -0
- package/dist/types/server.js.map +1 -0
- package/dist/types.d.ts +1 -168
- package/dist/types.d.ts.map +1 -1
- package/package.json +32 -2
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { toString } from "../utils/values.js";
|
|
4
|
+
import { getStorageBackend } from "./backend/index.js";
|
|
5
|
+
import { collectPartitionRecords } from "./partitions/records.js";
|
|
6
|
+
import { resolveDir } from "./partitions/internal.js";
|
|
7
|
+
import { sanitizePartitionName } from "./names.js";
|
|
8
|
+
function normalizeFormat(input) {
|
|
9
|
+
return input === "zip" ? "zip" : "tar.gz";
|
|
10
|
+
}
|
|
11
|
+
function stripArchiveExtension(fileName) {
|
|
12
|
+
if (fileName.endsWith(".tar.gz"))
|
|
13
|
+
return fileName.slice(0, -7);
|
|
14
|
+
if (fileName.endsWith(".zip"))
|
|
15
|
+
return fileName.slice(0, -4);
|
|
16
|
+
return fileName;
|
|
17
|
+
}
|
|
18
|
+
function normalizeOutputPath(options) {
|
|
19
|
+
const rawOutputPath = toString(options.outputPath).trim();
|
|
20
|
+
if (!rawOutputPath)
|
|
21
|
+
throw new Error("missing-export-output-path");
|
|
22
|
+
const format = normalizeFormat(options.format);
|
|
23
|
+
let outputPath = path.resolve(rawOutputPath);
|
|
24
|
+
if (!outputPath.endsWith(".tar.gz") && !outputPath.endsWith(".zip")) {
|
|
25
|
+
outputPath = `${outputPath}${format === "zip" ? ".zip" : ".tar.gz"}`;
|
|
26
|
+
}
|
|
27
|
+
if (format === "zip" && !outputPath.endsWith(".zip"))
|
|
28
|
+
throw new Error("export-format-output-extension-mismatch");
|
|
29
|
+
if (format === "tar.gz" && !outputPath.endsWith(".tar.gz"))
|
|
30
|
+
throw new Error("export-format-output-extension-mismatch");
|
|
31
|
+
return {
|
|
32
|
+
outputPath,
|
|
33
|
+
format,
|
|
34
|
+
overwrite: options.overwrite === true,
|
|
35
|
+
rootName: stripArchiveExtension(path.basename(outputPath)),
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
function buildManifest(dir, snapshot) {
|
|
39
|
+
return {
|
|
40
|
+
version: 1,
|
|
41
|
+
generated_at: new Date().toISOString(),
|
|
42
|
+
dir,
|
|
43
|
+
partitions: snapshot.partitions.map((item) => item.name),
|
|
44
|
+
partition_items: snapshot.partitions.map((item) => ({
|
|
45
|
+
name: item.name,
|
|
46
|
+
total: item.total,
|
|
47
|
+
last_activity_at: item.lastActivityAt,
|
|
48
|
+
})),
|
|
49
|
+
total: snapshot.total,
|
|
50
|
+
files: snapshot.files.map((file) => ({
|
|
51
|
+
path: file.path,
|
|
52
|
+
partition: file.partition,
|
|
53
|
+
groupKey: file.groupKey,
|
|
54
|
+
day: file.day,
|
|
55
|
+
hour: file.hour,
|
|
56
|
+
level: file.level,
|
|
57
|
+
compressed: file.compressed,
|
|
58
|
+
bytes: file.bytes,
|
|
59
|
+
rows: file.rows,
|
|
60
|
+
})),
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
async function resolvePartitions(dir, options) {
|
|
64
|
+
const records = await collectPartitionRecords(dir);
|
|
65
|
+
const allNames = records.map((item) => item.name).sort((a, b) => a.localeCompare(b));
|
|
66
|
+
if (!(options && Object.prototype.hasOwnProperty.call(options, "partitions")))
|
|
67
|
+
return allNames;
|
|
68
|
+
const requested = Array.from(new Set((options?.partitions || []).map((item) => sanitizePartitionName(item))));
|
|
69
|
+
if (!requested.length)
|
|
70
|
+
throw new Error("export-partitions-empty");
|
|
71
|
+
for (const name of requested) {
|
|
72
|
+
if (!allNames.includes(name))
|
|
73
|
+
throw new Error(`partition-not-found: ${name}`);
|
|
74
|
+
}
|
|
75
|
+
return requested.sort((a, b) => a.localeCompare(b));
|
|
76
|
+
}
|
|
77
|
+
async function exportPartitions(dir, options) {
|
|
78
|
+
const baseDir = resolveDir(dir);
|
|
79
|
+
if (!baseDir)
|
|
80
|
+
throw new Error("missing-log-dir");
|
|
81
|
+
const normalized = normalizeOutputPath(options);
|
|
82
|
+
const partitions = await resolvePartitions(baseDir, options);
|
|
83
|
+
const backend = getStorageBackend();
|
|
84
|
+
const snapshot = await backend.scanPartitions(baseDir, partitions);
|
|
85
|
+
const manifest = buildManifest(baseDir, snapshot);
|
|
86
|
+
if (fs.existsSync(normalized.outputPath) && !normalized.overwrite) {
|
|
87
|
+
throw new Error(`export-output-already-exists: ${normalized.outputPath}`);
|
|
88
|
+
}
|
|
89
|
+
const generatedFiles = [
|
|
90
|
+
{
|
|
91
|
+
archivePath: "manifest.json",
|
|
92
|
+
content: `${JSON.stringify(manifest, null, 2)}\n`,
|
|
93
|
+
},
|
|
94
|
+
];
|
|
95
|
+
await backend.createArchive({
|
|
96
|
+
outputPath: normalized.outputPath,
|
|
97
|
+
format: normalized.format,
|
|
98
|
+
rootName: normalized.rootName,
|
|
99
|
+
overwrite: normalized.overwrite,
|
|
100
|
+
generatedFiles,
|
|
101
|
+
sourceFiles: snapshot.files.map((file) => ({
|
|
102
|
+
sourcePath: file.absPath,
|
|
103
|
+
archivePath: path.posix.join("logs", file.path),
|
|
104
|
+
})),
|
|
105
|
+
});
|
|
106
|
+
return {
|
|
107
|
+
path: normalized.outputPath,
|
|
108
|
+
format: normalized.format,
|
|
109
|
+
backend: backend.name,
|
|
110
|
+
partitions: manifest.partitions,
|
|
111
|
+
files: manifest.total.files,
|
|
112
|
+
logs: manifest.total.logs,
|
|
113
|
+
bytes: manifest.total.bytes,
|
|
114
|
+
manifest,
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
async function exportPartition(dir, partition, options) {
|
|
118
|
+
return exportPartitions(dir, {
|
|
119
|
+
...options,
|
|
120
|
+
partitions: [sanitizePartitionName(partition)],
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
export { exportPartition, exportPartitions };
|
|
124
|
+
//# sourceMappingURL=export.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"export.js","sourceRoot":"","sources":["../../src/storage/export.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAG7B,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,uBAAuB,EAAE,MAAM,yBAAyB,CAAC;AAClE,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC;AASnD,SAAS,eAAe,CAAC,KAAc;IACrC,OAAO,KAAK,KAAK,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC;AAC5C,CAAC;AAED,SAAS,qBAAqB,CAAC,QAAgB;IAC7C,IAAI,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC;QAAE,OAAO,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/D,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,OAAO,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC5D,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,mBAAmB,CAAC,OAAyD;IACpF,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,CAAC;IAC1D,IAAI,CAAC,aAAa;QAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;IAClE,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC/C,IAAI,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;IAE7C,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QACpE,UAAU,GAAG,GAAG,UAAU,GAAG,MAAM,KAAK,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;IACvE,CAAC;IAED,IAAI,MAAM,KAAK,KAAK,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IACjH,IAAI,MAAM,KAAK,QAAQ,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,SAAS,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAEvH,OAAO;QACL,UAAU;QACV,MAAM;QACN,SAAS,EAAE,OAAO,CAAC,SAAS,KAAK,IAAI;QACrC,QAAQ,EAAE,qBAAqB,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;KAC3D,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAC,GAAW,EAAE,QAAqF;IACvH,OAAO;QACL,OAAO,EAAE,CAAC;QACV,YAAY,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACtC,GAAG;QACH,UAAU,EAAE,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC;QACxD,eAAe,EAAE,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAClD,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,gBAAgB,EAAE,IAAI,CAAC,cAAc;SACtC,CAAC,CAAC;QACH,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YACnC,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,IAAI,EAAE,IAAI,CAAC,IAAI;SAChB,CAAC,CAAC;KACJ,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,GAAW,EAAE,OAAiC;IAC7E,MAAM,OAAO,GAAG,MAAM,uBAAuB,CAAC,GAAG,CAAC,CAAC;IACnD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;IACrF,IAAI,CAAC,CAAC,OAAO,IAAI,MAAM,CAAC,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;QAAE,OAAO,QAAQ,CAAC;IAE/F,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,UAAU,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9G,IAAI,CAAC,SAAS,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;IAClE,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,IAAI,EAAE,CAAC,CAAC;IAChF,CAAC;IACD,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;AACtD,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,GAAW,EAAE,OAAgC;IAC3E,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;IAChC,IAAI,CAAC,OAAO;QAAE,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;IACjD,MAAM,UAAU,GAAG,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAChD,MAAM,UAAU,GAAG,MAAM,iBAAiB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC7D,MAAM,OAAO,GAAG,iBAAiB,EAAE,CAAC;IACpC,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,cAAc,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;IACnE,MAAM,QAAQ,GAAG,aAAa,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IAElD,IAAI,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,CAAC;QAClE,MAAM,IAAI,KAAK,CAAC,iCAAiC,UAAU,CAAC,UAAU,EAAE,CAAC,CAAC;IAC5E,CAAC;IAED,MAAM,cAAc,GAAG;QACrB;YACE,WAAW,EAAE,eAAe;YAC5B,OAAO,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI;SAClD;KACF,CAAC;IAEF,MAAM,OAAO,CAAC,aAAa,CAAC;QAC1B,UAAU,EAAE,UAAU,CAAC,UAAU;QACjC,MAAM,EAAE,UAAU,CAAC,MAAM;QACzB,QAAQ,EAAE,UAAU,CAAC,QAAQ;QAC7B,SAAS,EAAE,UAAU,CAAC,SAAS;QAC/B,cAAc;QACd,WAAW,EAAE,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YACzC,UAAU,EAAE,IAAI,CAAC,OAAO;YACxB,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC;SAChD,CAAC,CAAC;KACJ,CAAC,CAAC;IAEH,OAAO;QACL,IAAI,EAAE,UAAU,CAAC,UAAU;QAC3B,MAAM,EAAE,UAAU,CAAC,MAAM;QACzB,OAAO,EAAE,OAAO,CAAC,IAAI;QACrB,UAAU,EAAE,QAAQ,CAAC,UAAU;QAC/B,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,KAAK;QAC3B,IAAI,EAAE,QAAQ,CAAC,KAAK,CAAC,IAAI;QACzB,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,KAAK;QAC3B,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,eAAe,CAAC,GAAW,EAAE,SAAiB,EAAE,OAA+B;IAC5F,OAAO,gBAAgB,CAAC,GAAG,EAAE;QAC3B,GAAG,OAAO;QACV,UAAU,EAAE,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC;KAC/C,CAAC,CAAC;AACL,CAAC;AAED,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,CAAC"}
|
package/dist/storage/names.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { LogEntry } from "../types.js";
|
|
1
|
+
import type { LogEntry, PartitionNameOptions, PartitionSanitizeOptions } from "../types.js";
|
|
2
2
|
type ParsedLogFile = {
|
|
3
3
|
day: string;
|
|
4
4
|
hour: string;
|
|
@@ -13,12 +13,16 @@ type WalkedLogFile = ParsedLogFile & {
|
|
|
13
13
|
groupKey: string;
|
|
14
14
|
partition: string | null;
|
|
15
15
|
};
|
|
16
|
+
declare function sanitizePartitionName(input: string, options?: PartitionSanitizeOptions): string;
|
|
17
|
+
declare function formatPartitionTimePrefix(options?: PartitionNameOptions): string;
|
|
18
|
+
declare function buildPartitionName(options?: PartitionNameOptions): string;
|
|
19
|
+
declare function buildTemporaryPartitionName(options?: PartitionNameOptions): string;
|
|
16
20
|
declare function normalizePartitionKey(input: unknown): string;
|
|
17
21
|
declare function nowFileStamp(date?: Date, timeZone?: string): string;
|
|
18
22
|
declare function fileStampForEntry(entry: LogEntry, timeZone?: string): string;
|
|
19
23
|
declare function makeLogFileName(stamp: string, sequence: number, level: string): string;
|
|
20
24
|
declare function parseLogFileName(fileName: string): ParsedLogFile | null;
|
|
21
25
|
declare function walkedFileFromPath(baseDir: string, filePath: string, partition?: string | null, rootDir?: string): WalkedLogFile | null;
|
|
22
|
-
export { fileStampForEntry, makeLogFileName, normalizePartitionKey, nowFileStamp, parseLogFileName, walkedFileFromPath };
|
|
26
|
+
export { buildPartitionName, buildTemporaryPartitionName, fileStampForEntry, formatPartitionTimePrefix, makeLogFileName, normalizePartitionKey, nowFileStamp, parseLogFileName, sanitizePartitionName, walkedFileFromPath, };
|
|
23
27
|
export type { ParsedLogFile, WalkedLogFile };
|
|
24
28
|
//# sourceMappingURL=names.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"names.d.ts","sourceRoot":"","sources":["../../src/storage/names.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"names.d.ts","sourceRoot":"","sources":["../../src/storage/names.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EACV,QAAQ,EACR,oBAAoB,EACpB,wBAAwB,EAGzB,MAAM,aAAa,CAAC;AAIrB,KAAK,aAAa,GAAG;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,OAAO,CAAC;CACrB,CAAC;AAEF,KAAK,aAAa,GAAG,aAAa,GAAG;IACnC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B,CAAC;AAmBF,iBAAS,qBAAqB,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,wBAA6B,GAAG,MAAM,CAE5F;AAMD,iBAAS,yBAAyB,CAAC,OAAO,GAAE,oBAAyB,GAAG,MAAM,CAG7E;AAED,iBAAS,kBAAkB,CAAC,OAAO,GAAE,oBAAyB,GAAG,MAAM,CAYtE;AAED,iBAAS,2BAA2B,CAAC,OAAO,GAAE,oBAAyB,GAAG,MAAM,CAa/E;AAED,iBAAS,qBAAqB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAIrD;AAED,iBAAS,YAAY,CAAC,IAAI,OAAa,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,CAGlE;AAED,iBAAS,iBAAiB,CAAC,KAAK,EAAE,QAAQ,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,CAGrE;AAED,iBAAS,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAG/E;AAED,iBAAS,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI,CAWhE;AAED,iBAAS,kBAAkB,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,GAAE,MAAM,GAAG,IAAW,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI,CAetI;AAED,OAAO,EACL,kBAAkB,EAClB,2BAA2B,EAC3B,iBAAiB,EACjB,yBAAyB,EACzB,eAAe,EACf,qBAAqB,EACrB,YAAY,EACZ,gBAAgB,EAChB,qBAAqB,EACrB,kBAAkB,GACnB,CAAC;AACF,YAAY,EAAE,aAAa,EAAE,aAAa,EAAE,CAAC"}
|
package/dist/storage/names.js
CHANGED
|
@@ -1,13 +1,68 @@
|
|
|
1
|
+
import crypto from "node:crypto";
|
|
1
2
|
import path from "node:path";
|
|
2
3
|
import { TOP_LEVEL } from "../constants.js";
|
|
3
4
|
import { groupKeyFromRelDir } from "../groups.js";
|
|
4
|
-
import { toString } from "../utils/values.js";
|
|
5
5
|
import { getLocalDateTimeParts, normalizeTimeZone } from "../utils/datetime.js";
|
|
6
|
+
import { toString } from "../utils/values.js";
|
|
7
|
+
function sanitizePartitionFragment(input) {
|
|
8
|
+
return toString(input)
|
|
9
|
+
.trim()
|
|
10
|
+
.replace(/[^a-zA-Z0-9_-]+/g, "-")
|
|
11
|
+
.replace(/-+/g, "-")
|
|
12
|
+
.replace(/_+/g, "_")
|
|
13
|
+
.replace(/^[-_]+|[-_]+$/g, "");
|
|
14
|
+
}
|
|
15
|
+
function normalizePartitionValue(input, sanitizer) {
|
|
16
|
+
const raw = toString(input);
|
|
17
|
+
const prepared = typeof sanitizer === "function" ? sanitizer(raw) : raw;
|
|
18
|
+
const normalized = sanitizePartitionFragment(prepared);
|
|
19
|
+
if (!normalized)
|
|
20
|
+
throw new Error("invalid-partition-name");
|
|
21
|
+
return normalized;
|
|
22
|
+
}
|
|
23
|
+
function sanitizePartitionName(input, options = {}) {
|
|
24
|
+
return normalizePartitionValue(input, options.sanitizer);
|
|
25
|
+
}
|
|
26
|
+
function partitionTimeParts(at, timeZone) {
|
|
27
|
+
return getLocalDateTimeParts(at, normalizeTimeZone(timeZone));
|
|
28
|
+
}
|
|
29
|
+
function formatPartitionTimePrefix(options = {}) {
|
|
30
|
+
const parts = partitionTimeParts(options.at, options.timeZone);
|
|
31
|
+
return `${parts.year}-${parts.month}-${parts.day}-${parts.hour}-0000`;
|
|
32
|
+
}
|
|
33
|
+
function buildPartitionName(options = {}) {
|
|
34
|
+
const prefix = formatPartitionTimePrefix(options);
|
|
35
|
+
const suffix = toString(options.suffix).trim();
|
|
36
|
+
if (!suffix)
|
|
37
|
+
return prefix;
|
|
38
|
+
const preparedSuffix = options.sanitizeSuffix === false
|
|
39
|
+
? suffix
|
|
40
|
+
: typeof options.sanitizeSuffix === "function"
|
|
41
|
+
? options.sanitizeSuffix(suffix)
|
|
42
|
+
: suffix;
|
|
43
|
+
return sanitizePartitionName(`${prefix}-${preparedSuffix}`);
|
|
44
|
+
}
|
|
45
|
+
function buildTemporaryPartitionName(options = {}) {
|
|
46
|
+
const suffix = toString(options.suffix).trim();
|
|
47
|
+
const random = crypto.randomBytes(4).toString("hex");
|
|
48
|
+
const prefix = formatPartitionTimePrefix(options);
|
|
49
|
+
if (!suffix)
|
|
50
|
+
return sanitizePartitionName(`${prefix}-tmp-${random}`);
|
|
51
|
+
const preparedSuffix = options.sanitizeSuffix === false
|
|
52
|
+
? suffix
|
|
53
|
+
: typeof options.sanitizeSuffix === "function"
|
|
54
|
+
? options.sanitizeSuffix(suffix)
|
|
55
|
+
: suffix;
|
|
56
|
+
return sanitizePartitionName(`${prefix}-${preparedSuffix}-tmp-${random}`);
|
|
57
|
+
}
|
|
6
58
|
function normalizePartitionKey(input) {
|
|
7
|
-
|
|
59
|
+
const raw = toString(input).trim();
|
|
60
|
+
if (!raw)
|
|
61
|
+
return "";
|
|
62
|
+
return sanitizePartitionName(raw);
|
|
8
63
|
}
|
|
9
64
|
function nowFileStamp(date = new Date(), timeZone) {
|
|
10
|
-
const parts =
|
|
65
|
+
const parts = partitionTimeParts(date, timeZone);
|
|
11
66
|
return `${parts.year}-${parts.month}-${parts.day}-${parts.hour}`;
|
|
12
67
|
}
|
|
13
68
|
function fileStampForEntry(entry, timeZone) {
|
|
@@ -19,35 +74,15 @@ function makeLogFileName(stamp, sequence, level) {
|
|
|
19
74
|
return `${stamp}-${seq}-${level}.jsonl`;
|
|
20
75
|
}
|
|
21
76
|
function parseLogFileName(fileName) {
|
|
22
|
-
const
|
|
23
|
-
if (
|
|
24
|
-
return {
|
|
25
|
-
day: next[1],
|
|
26
|
-
hour: next[2],
|
|
27
|
-
sequence: Number(next[3]) || 0,
|
|
28
|
-
level: next[4],
|
|
29
|
-
compressed: Boolean(next[5]),
|
|
30
|
-
};
|
|
31
|
-
}
|
|
32
|
-
const legacyWithSequence = /^(\d{2}-\d{2}-\d{4})-(\d{2})-(\d+)-([a-z0-9._-]+)\.log\.json(\.gz)?$/i.exec(fileName);
|
|
33
|
-
if (legacyWithSequence) {
|
|
34
|
-
return {
|
|
35
|
-
day: legacyWithSequence[1],
|
|
36
|
-
hour: legacyWithSequence[2],
|
|
37
|
-
sequence: Number(legacyWithSequence[3]) || 0,
|
|
38
|
-
level: legacyWithSequence[4],
|
|
39
|
-
compressed: Boolean(legacyWithSequence[5]),
|
|
40
|
-
};
|
|
41
|
-
}
|
|
42
|
-
const legacy = /^(\d{2}-\d{2}-\d{4})-(\d{2})-([a-z0-9._-]+)\.log\.json(\.gz)?$/i.exec(fileName);
|
|
43
|
-
if (!legacy)
|
|
77
|
+
const match = /^(\d{4}-\d{2}-\d{2})-(\d{2})-(\d+)-([a-z0-9._-]+)\.jsonl(\.gz)?$/i.exec(fileName);
|
|
78
|
+
if (!match)
|
|
44
79
|
return null;
|
|
45
80
|
return {
|
|
46
|
-
day:
|
|
47
|
-
hour:
|
|
48
|
-
sequence: 0,
|
|
49
|
-
level:
|
|
50
|
-
compressed: Boolean(
|
|
81
|
+
day: match[1],
|
|
82
|
+
hour: match[2],
|
|
83
|
+
sequence: Number(match[3]) || 0,
|
|
84
|
+
level: match[4],
|
|
85
|
+
compressed: Boolean(match[5]),
|
|
51
86
|
};
|
|
52
87
|
}
|
|
53
88
|
function walkedFileFromPath(baseDir, filePath, partition = null, rootDir) {
|
|
@@ -66,5 +101,5 @@ function walkedFileFromPath(baseDir, filePath, partition = null, rootDir) {
|
|
|
66
101
|
partition,
|
|
67
102
|
};
|
|
68
103
|
}
|
|
69
|
-
export { fileStampForEntry, makeLogFileName, normalizePartitionKey, nowFileStamp, parseLogFileName, walkedFileFromPath };
|
|
104
|
+
export { buildPartitionName, buildTemporaryPartitionName, fileStampForEntry, formatPartitionTimePrefix, makeLogFileName, normalizePartitionKey, nowFileStamp, parseLogFileName, sanitizePartitionName, walkedFileFromPath, };
|
|
70
105
|
//# sourceMappingURL=names.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"names.js","sourceRoot":"","sources":["../../src/storage/names.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"names.js","sourceRoot":"","sources":["../../src/storage/names.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAQlD,OAAO,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAChF,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAkB9C,SAAS,yBAAyB,CAAC,KAAa;IAC9C,OAAO,QAAQ,CAAC,KAAK,CAAC;SACnB,IAAI,EAAE;SACN,OAAO,CAAC,kBAAkB,EAAE,GAAG,CAAC;SAChC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC;SACnB,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC;SACnB,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;AACnC,CAAC;AAED,SAAS,uBAAuB,CAAC,KAAc,EAAE,SAA8B;IAC7E,MAAM,GAAG,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC5B,MAAM,QAAQ,GAAG,OAAO,SAAS,KAAK,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IACxE,MAAM,UAAU,GAAG,yBAAyB,CAAC,QAAQ,CAAC,CAAC;IACvD,IAAI,CAAC,UAAU;QAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;IAC3D,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,SAAS,qBAAqB,CAAC,KAAa,EAAE,UAAoC,EAAE;IAClF,OAAO,uBAAuB,CAAC,KAAK,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,kBAAkB,CAAC,EAAkC,EAAE,QAAiB;IAC/E,OAAO,qBAAqB,CAAC,EAAE,EAAE,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC;AAChE,CAAC;AAED,SAAS,yBAAyB,CAAC,UAAgC,EAAE;IACnE,MAAM,KAAK,GAAG,kBAAkB,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC/D,OAAO,GAAG,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,GAAG,IAAI,KAAK,CAAC,IAAI,OAAO,CAAC;AACxE,CAAC;AAED,SAAS,kBAAkB,CAAC,UAAgC,EAAE;IAC5D,MAAM,MAAM,GAAG,yBAAyB,CAAC,OAAO,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/C,IAAI,CAAC,MAAM;QAAE,OAAO,MAAM,CAAC;IAE3B,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,KAAK,KAAK;QACrD,CAAC,CAAC,MAAM;QACR,CAAC,CAAC,OAAO,OAAO,CAAC,cAAc,KAAK,UAAU;YAC5C,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,MAAM,CAAC;YAChC,CAAC,CAAC,MAAM,CAAC;IAEb,OAAO,qBAAqB,CAAC,GAAG,MAAM,IAAI,cAAc,EAAE,CAAC,CAAC;AAC9D,CAAC;AAED,SAAS,2BAA2B,CAAC,UAAgC,EAAE;IACrE,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/C,MAAM,MAAM,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IACrD,MAAM,MAAM,GAAG,yBAAyB,CAAC,OAAO,CAAC,CAAC;IAClD,IAAI,CAAC,MAAM;QAAE,OAAO,qBAAqB,CAAC,GAAG,MAAM,QAAQ,MAAM,EAAE,CAAC,CAAC;IAErE,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,KAAK,KAAK;QACrD,CAAC,CAAC,MAAM;QACR,CAAC,CAAC,OAAO,OAAO,CAAC,cAAc,KAAK,UAAU;YAC5C,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,MAAM,CAAC;YAChC,CAAC,CAAC,MAAM,CAAC;IAEb,OAAO,qBAAqB,CAAC,GAAG,MAAM,IAAI,cAAc,QAAQ,MAAM,EAAE,CAAC,CAAC;AAC5E,CAAC;AAED,SAAS,qBAAqB,CAAC,KAAc;IAC3C,MAAM,GAAG,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;IACnC,IAAI,CAAC,GAAG;QAAE,OAAO,EAAE,CAAC;IACpB,OAAO,qBAAqB,CAAC,GAAG,CAAC,CAAC;AACpC,CAAC;AAED,SAAS,YAAY,CAAC,IAAI,GAAG,IAAI,IAAI,EAAE,EAAE,QAAiB;IACxD,MAAM,KAAK,GAAG,kBAAkB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IACjD,OAAO,GAAG,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;AACnE,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAe,EAAE,QAAiB;IAC3D,MAAM,IAAI,GAAG,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;IAC1E,OAAO,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;AAC5G,CAAC;AAED,SAAS,eAAe,CAAC,KAAa,EAAE,QAAgB,EAAE,KAAa;IACrE,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACvE,OAAO,GAAG,KAAK,IAAI,GAAG,IAAI,KAAK,QAAQ,CAAC;AAC1C,CAAC;AAED,SAAS,gBAAgB,CAAC,QAAgB;IACxC,MAAM,KAAK,GAAG,mEAAmE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACjG,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IAExB,OAAO;QACL,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC;QACb,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;QACd,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC/B,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;QACf,UAAU,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;KAC9B,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,OAAe,EAAE,QAAgB,EAAE,YAA2B,IAAI,EAAE,OAAgB;IAC9G,MAAM,MAAM,GAAG,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;IACzD,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IACzB,MAAM,YAAY,GAAG,OAAO,IAAI,OAAO,CAAC;IACxC,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;IACnE,MAAM,QAAQ,GAAG,MAAM,IAAI,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;IAExD,OAAO;QACL,GAAG,MAAM;QACT,OAAO,EAAE,QAAQ;QACjB,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACtD,QAAQ;QACR,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS;QAC7D,SAAS;KACV,CAAC;AACJ,CAAC;AAED,OAAO,EACL,kBAAkB,EAClB,2BAA2B,EAC3B,iBAAiB,EACjB,yBAAyB,EACzB,eAAe,EACf,qBAAqB,EACrB,YAAY,EACZ,gBAAgB,EAChB,qBAAqB,EACrB,kBAAkB,GACnB,CAAC"}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { DeleteLogsOptions, DeleteLogsResult, DeletePartitionResult, DeletePartitionsOptions } from "../../types.js";
|
|
2
|
+
declare function deletePartitions(dir: string, options?: DeletePartitionsOptions): Promise<DeletePartitionResult>;
|
|
3
|
+
declare function deleteLogs(dir: string, options?: DeleteLogsOptions): Promise<DeleteLogsResult>;
|
|
4
|
+
export { deleteLogs, deletePartitions };
|
|
5
|
+
//# sourceMappingURL=delete.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"delete.d.ts","sourceRoot":"","sources":["../../../src/storage/partitions/delete.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,qBAAqB,EAAE,uBAAuB,EAAE,MAAM,gBAAgB,CAAC;AAyE1H,iBAAe,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,uBAA4B,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAoBlH;AAED,iBAAe,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,iBAAsB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAqBjG;AAED,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,CAAC"}
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import { sanitizePartitionName } from "../names.js";
|
|
3
|
+
import { fileMatchesDeleteFilters, listTopLevelFiles, readLogRows } from "./files.js";
|
|
4
|
+
import { collectPartitionRecords, partitionInfoFromRecord, partitionMarkerMap } from "./records.js";
|
|
5
|
+
import { cutoffMsForDays, partitionAgeReferenceMs, resolveDir } from "./internal.js";
|
|
6
|
+
function sortPartitionCandidates(items) {
|
|
7
|
+
return items.sort((a, b) => a.info.name.localeCompare(b.info.name));
|
|
8
|
+
}
|
|
9
|
+
async function collectPartitionDeleteCandidates(dir, options = {}) {
|
|
10
|
+
const wanted = new Set((options.partitions || []).map((item) => sanitizePartitionName(item)));
|
|
11
|
+
const cutoff = cutoffMsForDays(options.olderThanDays);
|
|
12
|
+
const records = await collectPartitionRecords(dir);
|
|
13
|
+
const items = await Promise.all(records.map(async (record) => ({
|
|
14
|
+
record,
|
|
15
|
+
info: await partitionInfoFromRecord(record),
|
|
16
|
+
})));
|
|
17
|
+
return sortPartitionCandidates(items.filter(({ info }) => {
|
|
18
|
+
if (wanted.size && !wanted.has(info.name))
|
|
19
|
+
return false;
|
|
20
|
+
if (options.temporaryOnly === true && info.temporary !== true)
|
|
21
|
+
return false;
|
|
22
|
+
if (cutoff != null && partitionAgeReferenceMs(info) >= cutoff)
|
|
23
|
+
return false;
|
|
24
|
+
return true;
|
|
25
|
+
}));
|
|
26
|
+
}
|
|
27
|
+
async function fileDeleteCandidates(dir, options = {}) {
|
|
28
|
+
const baseDir = resolveDir(dir);
|
|
29
|
+
if (!baseDir)
|
|
30
|
+
return [];
|
|
31
|
+
const markers = await partitionMarkerMap(baseDir);
|
|
32
|
+
const files = await listTopLevelFiles(baseDir);
|
|
33
|
+
const cutoff = cutoffMsForDays(options.olderThanDays);
|
|
34
|
+
const hasExplicitPartition = Object.prototype.hasOwnProperty.call(options, "partition");
|
|
35
|
+
const explicitPartition = hasExplicitPartition && options.partition != null ? sanitizePartitionName(options.partition) : options.partition;
|
|
36
|
+
const acrossPartitions = options.acrossPartitions === true || !hasExplicitPartition;
|
|
37
|
+
const out = [];
|
|
38
|
+
for (const file of files) {
|
|
39
|
+
if (!fileMatchesDeleteFilters(file, options))
|
|
40
|
+
continue;
|
|
41
|
+
if (!acrossPartitions) {
|
|
42
|
+
if (explicitPartition == null) {
|
|
43
|
+
if (file.partition != null)
|
|
44
|
+
continue;
|
|
45
|
+
}
|
|
46
|
+
else if (file.partition !== explicitPartition)
|
|
47
|
+
continue;
|
|
48
|
+
}
|
|
49
|
+
else if (hasExplicitPartition) {
|
|
50
|
+
if (explicitPartition == null) {
|
|
51
|
+
if (file.partition != null)
|
|
52
|
+
continue;
|
|
53
|
+
}
|
|
54
|
+
else if (file.partition !== explicitPartition)
|
|
55
|
+
continue;
|
|
56
|
+
}
|
|
57
|
+
if (options.temporaryOnly === true) {
|
|
58
|
+
if (!file.partition)
|
|
59
|
+
continue;
|
|
60
|
+
if (markers.get(file.partition)?.temporary !== true)
|
|
61
|
+
continue;
|
|
62
|
+
}
|
|
63
|
+
let stat;
|
|
64
|
+
try {
|
|
65
|
+
stat = await fs.promises.stat(file.absPath);
|
|
66
|
+
}
|
|
67
|
+
catch {
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
if (cutoff != null && stat.mtimeMs >= cutoff)
|
|
71
|
+
continue;
|
|
72
|
+
out.push({
|
|
73
|
+
file,
|
|
74
|
+
bytes: stat.size,
|
|
75
|
+
logs: (await readLogRows(file.absPath, file.compressed)).length,
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
return out;
|
|
79
|
+
}
|
|
80
|
+
async function deletePartitions(dir, options = {}) {
|
|
81
|
+
const candidates = await collectPartitionDeleteCandidates(dir, options);
|
|
82
|
+
let files = 0;
|
|
83
|
+
let logs = 0;
|
|
84
|
+
let bytes = 0;
|
|
85
|
+
for (const item of candidates) {
|
|
86
|
+
files += item.info.total.files;
|
|
87
|
+
logs += item.info.total.logs;
|
|
88
|
+
bytes += item.info.total.bytes;
|
|
89
|
+
await fs.promises.rm(item.record.path, { recursive: true, force: true });
|
|
90
|
+
}
|
|
91
|
+
return {
|
|
92
|
+
partitions: candidates.length,
|
|
93
|
+
files,
|
|
94
|
+
logs,
|
|
95
|
+
bytes,
|
|
96
|
+
items: candidates.map((item) => item.info.name),
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
async function deleteLogs(dir, options = {}) {
|
|
100
|
+
const candidates = await fileDeleteCandidates(dir, options);
|
|
101
|
+
const partitions = new Set();
|
|
102
|
+
for (const item of candidates) {
|
|
103
|
+
if (item.file.partition)
|
|
104
|
+
partitions.add(item.file.partition);
|
|
105
|
+
await fs.promises.rm(item.file.absPath, { force: true });
|
|
106
|
+
}
|
|
107
|
+
return {
|
|
108
|
+
partitions: partitions.size,
|
|
109
|
+
files: candidates.length,
|
|
110
|
+
logs: candidates.reduce((sum, item) => sum + item.logs, 0),
|
|
111
|
+
bytes: candidates.reduce((sum, item) => sum + item.bytes, 0),
|
|
112
|
+
items: candidates.map((item) => ({
|
|
113
|
+
path: item.file.absPath,
|
|
114
|
+
partition: item.file.partition,
|
|
115
|
+
logs: item.logs,
|
|
116
|
+
bytes: item.bytes,
|
|
117
|
+
})),
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
export { deleteLogs, deletePartitions };
|
|
121
|
+
//# sourceMappingURL=delete.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"delete.js","sourceRoot":"","sources":["../../../src/storage/partitions/delete.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AAGzB,OAAO,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,EAAE,wBAAwB,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACtF,OAAO,EAAE,uBAAuB,EAAE,uBAAuB,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AACpG,OAAO,EAAE,eAAe,EAAE,uBAAuB,EAAE,UAAU,EAA0D,MAAM,eAAe,CAAC;AAE7I,SAAS,uBAAuB,CAAC,KAAiC;IAChE,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;AACtE,CAAC;AAED,KAAK,UAAU,gCAAgC,CAAC,GAAW,EAAE,UAAmC,EAAE;IAChG,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC9F,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;IACtD,MAAM,OAAO,GAAG,MAAM,uBAAuB,CAAC,GAAG,CAAC,CAAC;IACnD,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;QAC7D,MAAM;QACN,IAAI,EAAE,MAAM,uBAAuB,CAAC,MAAM,CAAC;KAC5C,CAAC,CAAC,CAAC,CAAC;IAEL,OAAO,uBAAuB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE;QACvD,IAAI,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,OAAO,KAAK,CAAC;QACxD,IAAI,OAAO,CAAC,aAAa,KAAK,IAAI,IAAI,IAAI,CAAC,SAAS,KAAK,IAAI;YAAE,OAAO,KAAK,CAAC;QAC5E,IAAI,MAAM,IAAI,IAAI,IAAI,uBAAuB,CAAC,IAAI,CAAC,IAAI,MAAM;YAAE,OAAO,KAAK,CAAC;QAC5E,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC,CAAC;AACN,CAAC;AAED,KAAK,UAAU,oBAAoB,CAAC,GAAW,EAAE,UAA6B,EAAE;IAC9E,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;IAChC,IAAI,CAAC,OAAO;QAAE,OAAO,EAAE,CAAC;IACxB,MAAM,OAAO,GAAG,MAAM,kBAAkB,CAAC,OAAO,CAAC,CAAC;IAClD,MAAM,KAAK,GAAG,MAAM,iBAAiB,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;IACtD,MAAM,oBAAoB,GAAG,MAAM,CAAC,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IACxF,MAAM,iBAAiB,GAAG,oBAAoB,IAAI,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,qBAAqB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC;IAC3I,MAAM,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,KAAK,IAAI,IAAI,CAAC,oBAAoB,CAAC;IAEpF,MAAM,GAAG,GAAyB,EAAE,CAAC;IACrC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,wBAAwB,CAAC,IAAI,EAAE,OAAO,CAAC;YAAE,SAAS;QACvD,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACtB,IAAI,iBAAiB,IAAI,IAAI,EAAE,CAAC;gBAC9B,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI;oBAAE,SAAS;YACvC,CAAC;iBAAM,IAAI,IAAI,CAAC,SAAS,KAAK,iBAAiB;gBAAE,SAAS;QAC5D,CAAC;aAAM,IAAI,oBAAoB,EAAE,CAAC;YAChC,IAAI,iBAAiB,IAAI,IAAI,EAAE,CAAC;gBAC9B,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI;oBAAE,SAAS;YACvC,CAAC;iBAAM,IAAI,IAAI,CAAC,SAAS,KAAK,iBAAiB;gBAAE,SAAS;QAC5D,CAAC;QAED,IAAI,OAAO,CAAC,aAAa,KAAK,IAAI,EAAE,CAAC;YACnC,IAAI,CAAC,IAAI,CAAC,SAAS;gBAAE,SAAS;YAC9B,IAAI,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,SAAS,KAAK,IAAI;gBAAE,SAAS;QAChE,CAAC;QAED,IAAI,IAAc,CAAC;QACnB,IAAI,CAAC;YACH,IAAI,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC9C,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QAED,IAAI,MAAM,IAAI,IAAI,IAAI,IAAI,CAAC,OAAO,IAAI,MAAM;YAAE,SAAS;QACvD,GAAG,CAAC,IAAI,CAAC;YACP,IAAI;YACJ,KAAK,EAAE,IAAI,CAAC,IAAI;YAChB,IAAI,EAAE,CAAC,MAAM,WAAW,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM;SAChE,CAAC,CAAC;IACL,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,GAAW,EAAE,UAAmC,EAAE;IAChF,MAAM,UAAU,GAAG,MAAM,gCAAgC,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IACxE,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;QAC/B,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;QAC7B,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;QAC/B,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3E,CAAC;IAED,OAAO;QACL,UAAU,EAAE,UAAU,CAAC,MAAM;QAC7B,KAAK;QACL,IAAI;QACJ,KAAK;QACL,KAAK,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;KAChD,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,GAAW,EAAE,UAA6B,EAAE;IACpE,MAAM,UAAU,GAAG,MAAM,oBAAoB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC5D,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;IAErC,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,IAAI,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC7D,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3D,CAAC;IAED,OAAO;QACL,UAAU,EAAE,UAAU,CAAC,IAAI;QAC3B,KAAK,EAAE,UAAU,CAAC,MAAM;QACxB,IAAI,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAC1D,KAAK,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;QAC5D,KAAK,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAC/B,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO;YACvB,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS;YAC9B,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB,CAAC,CAAC;KACJ,CAAC;AACJ,CAAC;AAED,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,CAAC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { DeleteLogsOptions, LogEntry } from "../../types.js";
|
|
2
|
+
import { type WalkedLogFile } from "../names.js";
|
|
3
|
+
import { type PartitionRecord } from "./internal.js";
|
|
4
|
+
declare function readLogRows(filePath: string, compressed: boolean): Promise<LogEntry[]>;
|
|
5
|
+
declare function writeLogRows(filePath: string, rows: LogEntry[], compressed: boolean): Promise<void>;
|
|
6
|
+
declare function collectPartitionFiles(record: PartitionRecord): Promise<WalkedLogFile[]>;
|
|
7
|
+
declare function findAvailableTargetPath(dir: string, file: WalkedLogFile): Promise<string>;
|
|
8
|
+
declare function writePartitionFiles(source: PartitionRecord, targetRoot: string, targetName: string, merge: boolean): Promise<void>;
|
|
9
|
+
declare function fileMatchesDeleteFilters(file: WalkedLogFile, options: DeleteLogsOptions): boolean;
|
|
10
|
+
declare function listTopLevelFiles(baseDir: string): Promise<WalkedLogFile[]>;
|
|
11
|
+
export { collectPartitionFiles, fileMatchesDeleteFilters, findAvailableTargetPath, listTopLevelFiles, readLogRows, writeLogRows, writePartitionFiles, };
|
|
12
|
+
//# sourceMappingURL=files.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"files.d.ts","sourceRoot":"","sources":["../../../src/storage/partitions/files.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,iBAAiB,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAGlE,OAAO,EAAuC,KAAK,aAAa,EAAE,MAAM,aAAa,CAAC;AACtF,OAAO,EAA+C,KAAK,eAAe,EAAE,MAAM,eAAe,CAAC;AAGlG,iBAAe,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAkBrF;AAED,iBAAe,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,UAAU,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAQlG;AAED,iBAAe,qBAAqB,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC,CA2BtF;AAED,iBAAe,uBAAuB,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAWxF;AAED,iBAAe,mBAAmB,CAAC,MAAM,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAajI;AAED,iBAAS,wBAAwB,CAAC,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAW1F;AAED,iBAAe,iBAAiB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC,CAgC1E;AAED,OAAO,EACL,qBAAqB,EACrB,wBAAwB,EACxB,uBAAuB,EACvB,iBAAiB,EACjB,WAAW,EACX,YAAY,EACZ,mBAAmB,GACpB,CAAC"}
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import zlib from "node:zlib";
|
|
4
|
+
import { PARTITION_MARKER_FILE } from "../../constants.js";
|
|
5
|
+
import { normGroup } from "../../groups.js";
|
|
6
|
+
import { toString } from "../../utils/values.js";
|
|
7
|
+
import { makeLogFileName, walkedFileFromPath } from "../names.js";
|
|
8
|
+
import { fileStamp, pathExists } from "./internal.js";
|
|
9
|
+
import { partitionMarkerMap } from "./records.js";
|
|
10
|
+
async function readLogRows(filePath, compressed) {
|
|
11
|
+
try {
|
|
12
|
+
const data = await fs.promises.readFile(filePath);
|
|
13
|
+
const text = compressed ? zlib.gunzipSync(data).toString("utf8") : data.toString("utf8");
|
|
14
|
+
if (!text.trim())
|
|
15
|
+
return [];
|
|
16
|
+
const rows = [];
|
|
17
|
+
for (const line of text.trim().split("\n")) {
|
|
18
|
+
try {
|
|
19
|
+
const parsed = JSON.parse(line);
|
|
20
|
+
if (parsed && typeof parsed === "object")
|
|
21
|
+
rows.push(parsed);
|
|
22
|
+
}
|
|
23
|
+
catch { }
|
|
24
|
+
}
|
|
25
|
+
return rows;
|
|
26
|
+
}
|
|
27
|
+
catch {
|
|
28
|
+
return [];
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
async function writeLogRows(filePath, rows, compressed) {
|
|
32
|
+
const payload = rows.length ? `${rows.map((row) => JSON.stringify(row)).join("\n")}\n` : "";
|
|
33
|
+
if (compressed) {
|
|
34
|
+
const zipped = zlib.gzipSync(Buffer.from(payload, "utf8"));
|
|
35
|
+
await fs.promises.writeFile(filePath, zipped);
|
|
36
|
+
return;
|
|
37
|
+
}
|
|
38
|
+
await fs.promises.writeFile(filePath, payload, "utf8");
|
|
39
|
+
}
|
|
40
|
+
async function collectPartitionFiles(record) {
|
|
41
|
+
const out = [];
|
|
42
|
+
const stack = [record.path];
|
|
43
|
+
while (stack.length) {
|
|
44
|
+
const current = stack.pop() || "";
|
|
45
|
+
let entries = [];
|
|
46
|
+
try {
|
|
47
|
+
entries = await fs.promises.readdir(current, { withFileTypes: true });
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
continue;
|
|
51
|
+
}
|
|
52
|
+
for (const entry of entries) {
|
|
53
|
+
const absPath = path.join(current, entry.name);
|
|
54
|
+
if (entry.isDirectory()) {
|
|
55
|
+
stack.push(absPath);
|
|
56
|
+
continue;
|
|
57
|
+
}
|
|
58
|
+
if (!entry.isFile() || entry.name === PARTITION_MARKER_FILE)
|
|
59
|
+
continue;
|
|
60
|
+
const walked = walkedFileFromPath(record.path, absPath, record.name, record.path);
|
|
61
|
+
if (walked)
|
|
62
|
+
out.push(walked);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
out.sort((a, b) => a.absPath.localeCompare(b.absPath));
|
|
66
|
+
return out;
|
|
67
|
+
}
|
|
68
|
+
async function findAvailableTargetPath(dir, file) {
|
|
69
|
+
let sequence = file.sequence;
|
|
70
|
+
for (;;) {
|
|
71
|
+
const fileName = makeLogFileName(fileStamp(file), sequence, file.level);
|
|
72
|
+
const plainTarget = path.join(dir, fileName);
|
|
73
|
+
const gzipTarget = path.join(dir, `${fileName}.gz`);
|
|
74
|
+
if (!(await pathExists(plainTarget)) && !(await pathExists(gzipTarget))) {
|
|
75
|
+
return file.compressed ? gzipTarget : plainTarget;
|
|
76
|
+
}
|
|
77
|
+
sequence += 1;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
async function writePartitionFiles(source, targetRoot, targetName, merge) {
|
|
81
|
+
const files = await collectPartitionFiles(source);
|
|
82
|
+
for (const file of files) {
|
|
83
|
+
const rows = await readLogRows(file.absPath, file.compressed);
|
|
84
|
+
const nextRows = rows.map((row) => ({ ...row, partition: targetName }));
|
|
85
|
+
const targetDir = file.groupDir ? path.join(targetRoot, file.groupDir) : targetRoot;
|
|
86
|
+
await fs.promises.mkdir(targetDir, { recursive: true });
|
|
87
|
+
const targetPath = merge
|
|
88
|
+
? await findAvailableTargetPath(targetDir, file)
|
|
89
|
+
: path.join(targetDir, path.basename(file.absPath));
|
|
90
|
+
await writeLogRows(targetPath, nextRows, file.compressed);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
function fileMatchesDeleteFilters(file, options) {
|
|
94
|
+
const level = toString(options.level || "all").toLowerCase() || "all";
|
|
95
|
+
const groupKey = toString(options.groupKey || "all") || "all";
|
|
96
|
+
const day = toString(options.day);
|
|
97
|
+
const hour = toString(options.hour);
|
|
98
|
+
if (day && file.day !== day)
|
|
99
|
+
return false;
|
|
100
|
+
if (hour && file.hour !== hour)
|
|
101
|
+
return false;
|
|
102
|
+
if (groupKey !== "all" && file.groupKey !== normGroup(groupKey).key)
|
|
103
|
+
return false;
|
|
104
|
+
if (level !== "all" && file.level !== level)
|
|
105
|
+
return false;
|
|
106
|
+
return true;
|
|
107
|
+
}
|
|
108
|
+
async function listTopLevelFiles(baseDir) {
|
|
109
|
+
const out = [];
|
|
110
|
+
const stack = [{ dir: baseDir, rootDir: baseDir, partition: null }];
|
|
111
|
+
const markers = await partitionMarkerMap(baseDir);
|
|
112
|
+
while (stack.length) {
|
|
113
|
+
const current = stack.pop();
|
|
114
|
+
let entries = [];
|
|
115
|
+
try {
|
|
116
|
+
entries = await fs.promises.readdir(current.dir, { withFileTypes: true });
|
|
117
|
+
}
|
|
118
|
+
catch {
|
|
119
|
+
continue;
|
|
120
|
+
}
|
|
121
|
+
for (const entry of entries) {
|
|
122
|
+
const absPath = path.join(current.dir, entry.name);
|
|
123
|
+
if (entry.isDirectory()) {
|
|
124
|
+
if (current.partition == null && markers.has(entry.name)) {
|
|
125
|
+
stack.push({ dir: absPath, rootDir: absPath, partition: entry.name });
|
|
126
|
+
continue;
|
|
127
|
+
}
|
|
128
|
+
stack.push({ dir: absPath, rootDir: current.rootDir, partition: current.partition });
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
if (!entry.isFile() || entry.name === PARTITION_MARKER_FILE)
|
|
132
|
+
continue;
|
|
133
|
+
const walked = walkedFileFromPath(current.rootDir, absPath, current.partition, current.rootDir);
|
|
134
|
+
if (walked)
|
|
135
|
+
out.push(walked);
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
out.sort((a, b) => a.absPath.localeCompare(b.absPath));
|
|
139
|
+
return out;
|
|
140
|
+
}
|
|
141
|
+
export { collectPartitionFiles, fileMatchesDeleteFilters, findAvailableTargetPath, listTopLevelFiles, readLogRows, writeLogRows, writePartitionFiles, };
|
|
142
|
+
//# sourceMappingURL=files.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"files.js","sourceRoot":"","sources":["../../../src/storage/partitions/files.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAE3D,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAsB,MAAM,aAAa,CAAC;AACtF,OAAO,EAAE,SAAS,EAAE,UAAU,EAA8C,MAAM,eAAe,CAAC;AAClG,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAElD,KAAK,UAAU,WAAW,CAAC,QAAgB,EAAE,UAAmB;IAC9D,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAClD,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACzF,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YAAE,OAAO,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAe,EAAE,CAAC;QAE5B,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3C,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAChC,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ;oBAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC9D,CAAC;YAAC,MAAM,CAAC,CAAA,CAAC;QACZ,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,QAAgB,EAAE,IAAgB,EAAE,UAAmB;IACjF,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5F,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;QAC3D,MAAM,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC9C,OAAO;IACT,CAAC;IACD,MAAM,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;AACzD,CAAC;AAED,KAAK,UAAU,qBAAqB,CAAC,MAAuB;IAC1D,MAAM,GAAG,GAAoB,EAAE,CAAC;IAChC,MAAM,KAAK,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAE5B,OAAO,KAAK,CAAC,MAAM,EAAE,CAAC;QACpB,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;QAClC,IAAI,OAAO,GAAgB,EAAE,CAAC;QAC9B,IAAI,CAAC;YACH,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QACxE,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QAED,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YAC/C,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;gBACxB,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACpB,SAAS;YACX,CAAC;YACD,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,KAAK,CAAC,IAAI,KAAK,qBAAqB;gBAAE,SAAS;YACtE,MAAM,MAAM,GAAG,kBAAkB,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;YAClF,IAAI,MAAM;gBAAE,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACvD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,KAAK,UAAU,uBAAuB,CAAC,GAAW,EAAE,IAAmB;IACrE,IAAI,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;IAC7B,SAAS,CAAC;QACR,MAAM,QAAQ,GAAG,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;QACxE,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;QAC7C,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,KAAK,CAAC,CAAC;QACpD,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;YACxE,OAAO,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,WAAW,CAAC;QACpD,CAAC;QACD,QAAQ,IAAI,CAAC,CAAC;IAChB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,MAAuB,EAAE,UAAkB,EAAE,UAAkB,EAAE,KAAc;IAChH,MAAM,KAAK,GAAG,MAAM,qBAAqB,CAAC,MAAM,CAAC,CAAC;IAElD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAC9D,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,GAAG,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC;QACxE,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;QACpF,MAAM,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACxD,MAAM,UAAU,GAAG,KAAK;YACtB,CAAC,CAAC,MAAM,uBAAuB,CAAC,SAAS,EAAE,IAAI,CAAC;YAChD,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;QACtD,MAAM,YAAY,CAAC,UAAU,EAAE,QAAQ,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;IAC5D,CAAC;AACH,CAAC;AAED,SAAS,wBAAwB,CAAC,IAAmB,EAAE,OAA0B;IAC/E,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,IAAI,KAAK,CAAC,CAAC,WAAW,EAAE,IAAI,KAAK,CAAC;IACtE,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,QAAQ,IAAI,KAAK,CAAC,IAAI,KAAK,CAAC;IAC9D,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAEpC,IAAI,GAAG,IAAI,IAAI,CAAC,GAAG,KAAK,GAAG;QAAE,OAAO,KAAK,CAAC;IAC1C,IAAI,IAAI,IAAI,IAAI,CAAC,IAAI,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAC7C,IAAI,QAAQ,KAAK,KAAK,IAAI,IAAI,CAAC,QAAQ,KAAK,SAAS,CAAC,QAAQ,CAAC,CAAC,GAAG;QAAE,OAAO,KAAK,CAAC;IAClF,IAAI,KAAK,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,KAAK;QAAE,OAAO,KAAK,CAAC;IAC1D,OAAO,IAAI,CAAC;AACd,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,OAAe;IAC9C,MAAM,GAAG,GAAoB,EAAE,CAAC;IAChC,MAAM,KAAK,GAAsE,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACvI,MAAM,OAAO,GAAG,MAAM,kBAAkB,CAAC,OAAO,CAAC,CAAC;IAElD,OAAO,KAAK,CAAC,MAAM,EAAE,CAAC;QACpB,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,EAAgE,CAAC;QAC1F,IAAI,OAAO,GAAgB,EAAE,CAAC;QAC9B,IAAI,CAAC;YACH,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5E,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QAED,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YACnD,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;gBACxB,IAAI,OAAO,CAAC,SAAS,IAAI,IAAI,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;oBACzD,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;oBACtE,SAAS;gBACX,CAAC;gBACD,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;gBACrF,SAAS;YACX,CAAC;YACD,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,KAAK,CAAC,IAAI,KAAK,qBAAqB;gBAAE,SAAS;YACtE,MAAM,MAAM,GAAG,kBAAkB,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;YAChG,IAAI,MAAM;gBAAE,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACvD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,OAAO,EACL,qBAAqB,EACrB,wBAAwB,EACxB,uBAAuB,EACvB,iBAAiB,EACjB,WAAW,EACX,YAAY,EACZ,mBAAmB,GACpB,CAAC"}
|