@hue-run/sdk 0.20.0 → 0.20.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI.md +102 -10
- package/ENVIRONMENTS.md +1 -1
- package/README.md +30 -2
- package/WORLDS.md +14 -3
- package/dist/cli/traces-import-convert.d.ts +262 -0
- package/dist/cli/traces-import-convert.js +1779 -0
- package/dist/cli/traces-import-prepare.d.ts +64 -0
- package/dist/cli/traces-import-prepare.js +443 -0
- package/dist/cli/traces-import-upload.d.ts +129 -0
- package/dist/cli/traces-import-upload.js +838 -0
- package/dist/cli/traces-import-zip.d.ts +52 -0
- package/dist/cli/traces-import-zip.js +278 -0
- package/dist/cli/traces-import.d.ts +18 -0
- package/dist/cli/traces-import.js +231 -0
- package/dist/setup/cli.js +7 -3
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/world/client.d.ts +13 -2
- package/dist/world/client.js +24 -5
- package/dist/world/model-types.d.ts +8 -2
- package/dist/world/models.d.ts +3 -1
- package/dist/world/readiness.d.ts +18 -0
- package/dist/world/readiness.js +69 -0
- package/dist/world/world.d.ts +3 -2
- package/dist/world/world.js +4 -2
- package/package.json +1 -1
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { type TraceImportFormat, type TraceImportManifest, type TraceImportOrigin, type TraceImportTotals } from "./traces-import-convert.js";
|
|
2
|
+
/**
|
|
3
|
+
* `hue traces import prepare`: reads a file, a directory or a ZIP archive, converts every trace
|
|
4
|
+
* it recognizes and writes the bundle Hue's import routes take: one gzip OTLP/JSON file per
|
|
5
|
+
* batch, one file per large value, and `manifest.json`, whose digest and totals
|
|
6
|
+
* `prepare_trace_import` is called with. Nothing is sent anywhere.
|
|
7
|
+
*/
|
|
8
|
+
/** Bounds on what one preparation reads. */
|
|
9
|
+
export declare const PREPARE_LIMITS: {
|
|
10
|
+
/** One source file, or one archive member. */
|
|
11
|
+
readonly fileBytes: number;
|
|
12
|
+
/** Every source file together. */
|
|
13
|
+
readonly totalBytes: number;
|
|
14
|
+
/** Files one directory tree or archive may hold. */
|
|
15
|
+
readonly files: 50000;
|
|
16
|
+
/** How deep a directory tree is walked. */
|
|
17
|
+
readonly depth: 32;
|
|
18
|
+
};
|
|
19
|
+
export declare const MANIFEST_FILE = "manifest.json";
|
|
20
|
+
export declare const PROGRESS_FILE = "progress.json";
|
|
21
|
+
export interface PrepareReport {
|
|
22
|
+
manifest_digest: string;
|
|
23
|
+
totals: TraceImportTotals;
|
|
24
|
+
source: TraceImportManifest["source"];
|
|
25
|
+
output_directory: string;
|
|
26
|
+
/** Traces recognized per format. */
|
|
27
|
+
formats: Record<TraceImportFormat, number>;
|
|
28
|
+
files: {
|
|
29
|
+
supported: {
|
|
30
|
+
file: string;
|
|
31
|
+
format: TraceImportFormat;
|
|
32
|
+
traces: number;
|
|
33
|
+
}[];
|
|
34
|
+
unsupported: {
|
|
35
|
+
file: string;
|
|
36
|
+
reason: string;
|
|
37
|
+
}[];
|
|
38
|
+
};
|
|
39
|
+
traces: {
|
|
40
|
+
without_identity: number;
|
|
41
|
+
by_origin: Record<TraceImportOrigin, number>;
|
|
42
|
+
/** Traces whose vendor ids were mapped to OpenTelemetry ids, and the spans so mapped. */
|
|
43
|
+
remapped: {
|
|
44
|
+
traces: number;
|
|
45
|
+
spans: number;
|
|
46
|
+
};
|
|
47
|
+
/** Traces and spans the sources held that could not be converted. */
|
|
48
|
+
refused: {
|
|
49
|
+
file?: string;
|
|
50
|
+
trace_id?: string;
|
|
51
|
+
reason: string;
|
|
52
|
+
}[];
|
|
53
|
+
};
|
|
54
|
+
blobs: number;
|
|
55
|
+
}
|
|
56
|
+
export declare class PrepareError extends Error {
|
|
57
|
+
}
|
|
58
|
+
/** The default output directory: `<input>.hue-import` beside the input. */
|
|
59
|
+
export declare function defaultOutputDirectory(input: string): string;
|
|
60
|
+
export interface PrepareOptions {
|
|
61
|
+
out?: string;
|
|
62
|
+
}
|
|
63
|
+
/** Prepares the bundle; resolves to the report, or throws a {@link PrepareError}. */
|
|
64
|
+
export declare function prepareTraceBundle(input: string, options?: PrepareOptions): Promise<PrepareReport>;
|
|
@@ -0,0 +1,443 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { mkdir, open, readdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises";
|
|
3
|
+
import { basename, join, relative, resolve, sep } from "node:path";
|
|
4
|
+
import { sdkVersion } from "../version.js";
|
|
5
|
+
import { batchEntries, ConversionError, convertAtifExport, detectFormat, LangfuseTraceCollector, manifestTotals, manifestTraceHasIdentity, OtlpTraceCollector, sha256Hex, traceImportFormats, traceImportLimits, validateManifest, } from "./traces-import-convert.js";
|
|
6
|
+
import { isZipSignature, ZipArchive, ZipError } from "./traces-import-zip.js";
|
|
7
|
+
/**
|
|
8
|
+
* `hue traces import prepare`: reads a file, a directory or a ZIP archive, converts every trace
|
|
9
|
+
* it recognizes and writes the bundle Hue's import routes take: one gzip OTLP/JSON file per
|
|
10
|
+
* batch, one file per large value, and `manifest.json`, whose digest and totals
|
|
11
|
+
* `prepare_trace_import` is called with. Nothing is sent anywhere.
|
|
12
|
+
*/
|
|
13
|
+
/** Bounds on what one preparation reads. */
|
|
14
|
+
export const PREPARE_LIMITS = {
|
|
15
|
+
/** One source file, or one archive member. */
|
|
16
|
+
fileBytes: 64 * 1024 * 1024,
|
|
17
|
+
/** Every source file together. */
|
|
18
|
+
totalBytes: 1024 * 1024 * 1024,
|
|
19
|
+
/** Files one directory tree or archive may hold. */
|
|
20
|
+
files: 50_000,
|
|
21
|
+
/** How deep a directory tree is walked. */
|
|
22
|
+
depth: 32,
|
|
23
|
+
};
|
|
24
|
+
export const MANIFEST_FILE = "manifest.json";
|
|
25
|
+
export const PROGRESS_FILE = "progress.json";
|
|
26
|
+
const BATCH_FILE = /^[0-9a-f]{32}\.[0-9a-f]{64}\.json\.gz$/;
|
|
27
|
+
const BLOB_FILE = /^[0-9a-f]{64}\.blob$/;
|
|
28
|
+
export class PrepareError extends Error {
|
|
29
|
+
}
|
|
30
|
+
const posixPath = (path) => path.split(sep).join("/");
|
|
31
|
+
async function readHead(path, length) {
|
|
32
|
+
const handle = await open(path, "r");
|
|
33
|
+
try {
|
|
34
|
+
const buffer = Buffer.alloc(length);
|
|
35
|
+
const { bytesRead } = await handle.read(buffer, 0, length, 0);
|
|
36
|
+
return buffer.subarray(0, bytesRead);
|
|
37
|
+
}
|
|
38
|
+
finally {
|
|
39
|
+
await handle.close();
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* The source files under `input`: a file, every regular file of a directory tree (symbolic links
|
|
44
|
+
* are listed and left unread), and every readable member of a ZIP archive found at either place.
|
|
45
|
+
* The walk is bounded and its order does not matter: the manifest is sorted by trace id.
|
|
46
|
+
*/
|
|
47
|
+
async function* sourceFiles(input, unsupported) {
|
|
48
|
+
const root = resolve(input);
|
|
49
|
+
const info = await stat(root);
|
|
50
|
+
let count = 0;
|
|
51
|
+
const seen = () => {
|
|
52
|
+
if (++count > PREPARE_LIMITS.files)
|
|
53
|
+
throw new PrepareError(`The input holds more than ${PREPARE_LIMITS.files} files`);
|
|
54
|
+
};
|
|
55
|
+
/** Why an entry the walk listed could not be opened: the system's code, when it gave one. */
|
|
56
|
+
const unreadable = (what, error) => {
|
|
57
|
+
const code = error.code;
|
|
58
|
+
return `the ${what} could not be read${code ? ` (${code})` : ""}`;
|
|
59
|
+
};
|
|
60
|
+
async function* file(path, name) {
|
|
61
|
+
seen();
|
|
62
|
+
let size;
|
|
63
|
+
let head;
|
|
64
|
+
try {
|
|
65
|
+
size = (await stat(path)).size;
|
|
66
|
+
head = await readHead(path, 4);
|
|
67
|
+
}
|
|
68
|
+
catch (error) {
|
|
69
|
+
// A file the walk can list but not open (permissions, a file removed meanwhile) is
|
|
70
|
+
// reported; the rest of the input is still prepared.
|
|
71
|
+
unsupported.push({ file: name, reason: unreadable("file", error) });
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
if (isZipSignature(head)) {
|
|
75
|
+
yield* archive(path, name);
|
|
76
|
+
return;
|
|
77
|
+
}
|
|
78
|
+
if (size > PREPARE_LIMITS.fileBytes) {
|
|
79
|
+
unsupported.push({ file: name, reason: `larger than ${PREPARE_LIMITS.fileBytes} bytes` });
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
yield { name, size, read: () => readFile(path) };
|
|
83
|
+
}
|
|
84
|
+
async function* archive(path, name) {
|
|
85
|
+
let zip;
|
|
86
|
+
try {
|
|
87
|
+
zip = await ZipArchive.open(path);
|
|
88
|
+
}
|
|
89
|
+
catch (error) {
|
|
90
|
+
unsupported.push({
|
|
91
|
+
file: name,
|
|
92
|
+
reason: error instanceof ZipError ? error.message : "the archive could not be read",
|
|
93
|
+
});
|
|
94
|
+
return;
|
|
95
|
+
}
|
|
96
|
+
// Each member is read while its archive is open; the archive is closed as soon as its last
|
|
97
|
+
// member was handed out (or the walk ended early), never held until the whole tree is read.
|
|
98
|
+
try {
|
|
99
|
+
for (const member of zip.members) {
|
|
100
|
+
if (member.path.endsWith("/"))
|
|
101
|
+
continue;
|
|
102
|
+
const memberName = `${name}/${member.path}`;
|
|
103
|
+
seen();
|
|
104
|
+
if (member.refused) {
|
|
105
|
+
unsupported.push({ file: memberName, reason: member.refused });
|
|
106
|
+
continue;
|
|
107
|
+
}
|
|
108
|
+
yield { name: memberName, size: member.size, read: () => zip.read(member) };
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
finally {
|
|
112
|
+
await zip.close().catch(() => { });
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
async function* directory(path, depth) {
|
|
116
|
+
if (depth > PREPARE_LIMITS.depth)
|
|
117
|
+
throw new PrepareError(`The directory tree is deeper than ${PREPARE_LIMITS.depth} levels`);
|
|
118
|
+
let entries;
|
|
119
|
+
try {
|
|
120
|
+
entries = await readdir(path, { withFileTypes: true });
|
|
121
|
+
}
|
|
122
|
+
catch (error) {
|
|
123
|
+
// The input itself must be readable; a subdirectory that is not is reported and skipped.
|
|
124
|
+
if (depth === 0)
|
|
125
|
+
throw error;
|
|
126
|
+
seen();
|
|
127
|
+
unsupported.push({
|
|
128
|
+
file: posixPath(relative(root, path)),
|
|
129
|
+
reason: unreadable("directory", error),
|
|
130
|
+
});
|
|
131
|
+
return;
|
|
132
|
+
}
|
|
133
|
+
entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
|
|
134
|
+
for (const entry of entries) {
|
|
135
|
+
const child = join(path, entry.name);
|
|
136
|
+
const name = posixPath(relative(root, child));
|
|
137
|
+
if (entry.isSymbolicLink()) {
|
|
138
|
+
seen();
|
|
139
|
+
unsupported.push({ file: name, reason: "a symbolic link" });
|
|
140
|
+
}
|
|
141
|
+
else if (entry.isDirectory())
|
|
142
|
+
yield* directory(child, depth + 1);
|
|
143
|
+
else if (entry.isFile())
|
|
144
|
+
yield* file(child, name);
|
|
145
|
+
else {
|
|
146
|
+
seen();
|
|
147
|
+
unsupported.push({ file: name, reason: "not a regular file" });
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
if (info.isDirectory())
|
|
152
|
+
yield* directory(root, 0);
|
|
153
|
+
else if (info.isFile())
|
|
154
|
+
yield* file(root, basename(root));
|
|
155
|
+
else
|
|
156
|
+
throw new PrepareError("The input is neither a file nor a directory");
|
|
157
|
+
}
|
|
158
|
+
/** The default output directory: `<input>.hue-import` beside the input. */
|
|
159
|
+
export function defaultOutputDirectory(input) {
|
|
160
|
+
const path = resolve(input);
|
|
161
|
+
return `${path.endsWith(sep) ? path.slice(0, -1) : path}.hue-import`;
|
|
162
|
+
}
|
|
163
|
+
/** Creates the output directory (mode 0700), or accepts one that holds only an earlier
|
|
164
|
+
* preparation's files; a directory an upload has started in is refused. */
|
|
165
|
+
async function prepareOutputDirectory(directory) {
|
|
166
|
+
let existing;
|
|
167
|
+
try {
|
|
168
|
+
existing = await readdir(directory);
|
|
169
|
+
}
|
|
170
|
+
catch (error) {
|
|
171
|
+
if (error.code !== "ENOENT")
|
|
172
|
+
throw error;
|
|
173
|
+
await mkdir(directory, { recursive: true, mode: 0o700 });
|
|
174
|
+
return;
|
|
175
|
+
}
|
|
176
|
+
if (existing.includes(PROGRESS_FILE))
|
|
177
|
+
throw new PrepareError(`${directory} holds an upload in progress (${PROGRESS_FILE}); choose another --out`);
|
|
178
|
+
const own = (name) => name === MANIFEST_FILE || BATCH_FILE.test(name) || BLOB_FILE.test(name);
|
|
179
|
+
/** A temporary file this helper writes before its rename (`<file>.tmp`), left behind when a
|
|
180
|
+
* preparation or an upload was interrupted between the two. */
|
|
181
|
+
const ownTemporary = (name) => name.endsWith(".tmp") && (own(name.slice(0, -4)) || name.slice(0, -4) === PROGRESS_FILE);
|
|
182
|
+
const foreign = existing.filter((name) => !own(name) && !ownTemporary(name));
|
|
183
|
+
if (foreign.length)
|
|
184
|
+
throw new PrepareError(`${directory} is not empty (${foreign[0]}); choose another --out`);
|
|
185
|
+
for (const name of existing.filter(ownTemporary)) {
|
|
186
|
+
const path = join(directory, name);
|
|
187
|
+
const regular = await stat(path)
|
|
188
|
+
.then((info) => info.isFile())
|
|
189
|
+
.catch(() => false);
|
|
190
|
+
if (!regular)
|
|
191
|
+
throw new PrepareError(`${directory} is not empty (${name}); choose another --out`);
|
|
192
|
+
// The file it stood for is written again from scratch.
|
|
193
|
+
await rm(path, { force: true });
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
async function writePrivate(path, bytes) {
|
|
197
|
+
const temporary = `${path}.tmp`;
|
|
198
|
+
await writeFile(temporary, bytes, { mode: 0o600, flag: "w" });
|
|
199
|
+
await rename(temporary, path);
|
|
200
|
+
}
|
|
201
|
+
/** Merges a converted trace into the collection: the same trace from two sources is one trace
|
|
202
|
+
* when its entries agree, a conflict otherwise. */
|
|
203
|
+
function merge(collected, conflicts, trace, sourceDigest) {
|
|
204
|
+
trace.sourceDigests.add(sourceDigest);
|
|
205
|
+
const held = collected.get(trace.traceId);
|
|
206
|
+
if (!held) {
|
|
207
|
+
collected.set(trace.traceId, trace);
|
|
208
|
+
return;
|
|
209
|
+
}
|
|
210
|
+
if (JSON.stringify(held.entries) !== JSON.stringify(trace.entries)) {
|
|
211
|
+
conflicts.set(trace.traceId, "the trace appears in two sources with different content");
|
|
212
|
+
return;
|
|
213
|
+
}
|
|
214
|
+
for (const digest of trace.sourceDigests)
|
|
215
|
+
held.sourceDigests.add(digest);
|
|
216
|
+
}
|
|
217
|
+
/** Prepares the bundle; resolves to the report, or throws a {@link PrepareError}. */
|
|
218
|
+
export async function prepareTraceBundle(input, options = {}) {
|
|
219
|
+
const outputDirectory = resolve(options.out ?? defaultOutputDirectory(input));
|
|
220
|
+
const unsupported = [];
|
|
221
|
+
const supported = [];
|
|
222
|
+
const refused = [];
|
|
223
|
+
const otlp = new OtlpTraceCollector();
|
|
224
|
+
const langfuse = new LangfuseTraceCollector();
|
|
225
|
+
const whole = new Map();
|
|
226
|
+
const conflicts = new Map();
|
|
227
|
+
/** The file names each source digest was read under, so the traces an OTLP or Langfuse file
|
|
228
|
+
* held are counted once the traces are known: a trace may come from several files. */
|
|
229
|
+
const fileNames = new Map();
|
|
230
|
+
const tracesPerFile = new Map();
|
|
231
|
+
const nameFile = (digest, name) => {
|
|
232
|
+
const names = fileNames.get(digest) ?? new Set();
|
|
233
|
+
names.add(name);
|
|
234
|
+
fileNames.set(digest, names);
|
|
235
|
+
};
|
|
236
|
+
let totalBytes = 0;
|
|
237
|
+
{
|
|
238
|
+
for await (const source of sourceFiles(input, unsupported)) {
|
|
239
|
+
totalBytes += source.size;
|
|
240
|
+
if (totalBytes > PREPARE_LIMITS.totalBytes)
|
|
241
|
+
throw new PrepareError(`The input holds more than ${PREPARE_LIMITS.totalBytes} bytes`);
|
|
242
|
+
let bytes;
|
|
243
|
+
try {
|
|
244
|
+
bytes = await source.read();
|
|
245
|
+
}
|
|
246
|
+
catch (error) {
|
|
247
|
+
unsupported.push({
|
|
248
|
+
file: source.name,
|
|
249
|
+
reason: error instanceof ZipError ? error.message : "the file could not be read",
|
|
250
|
+
});
|
|
251
|
+
continue;
|
|
252
|
+
}
|
|
253
|
+
const digest = sha256Hex(bytes);
|
|
254
|
+
const detected = detectFormat(bytes);
|
|
255
|
+
if (detected.format === "unsupported") {
|
|
256
|
+
unsupported.push({ file: source.name, reason: detected.reason });
|
|
257
|
+
continue;
|
|
258
|
+
}
|
|
259
|
+
const note = (problems) => {
|
|
260
|
+
for (const problem of problems)
|
|
261
|
+
refused.push({
|
|
262
|
+
file: source.name,
|
|
263
|
+
...(problem.traceId ? { trace_id: problem.traceId } : {}),
|
|
264
|
+
reason: problem.reason,
|
|
265
|
+
});
|
|
266
|
+
};
|
|
267
|
+
try {
|
|
268
|
+
if (detected.format === "otlp_json") {
|
|
269
|
+
note(otlp.add(detected.envelopes, digest));
|
|
270
|
+
nameFile(digest, source.name);
|
|
271
|
+
supported.push({ file: source.name, format: "otlp_json", traces: 0 });
|
|
272
|
+
}
|
|
273
|
+
else if (detected.format === "langfuse_observations") {
|
|
274
|
+
if (detected.unsafeIntegers)
|
|
275
|
+
throw new ConversionError("Source JSON contains an unsafe number");
|
|
276
|
+
langfuse.add(detected.observations, detected.envelope, digest);
|
|
277
|
+
nameFile(digest, source.name);
|
|
278
|
+
supported.push({ file: source.name, format: "langfuse_observations", traces: 0 });
|
|
279
|
+
}
|
|
280
|
+
else {
|
|
281
|
+
const trace = convertAtifExport(detected, bytes.byteLength);
|
|
282
|
+
merge(whole, conflicts, trace, digest);
|
|
283
|
+
supported.push({ file: source.name, format: "atif", traces: 1 });
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
catch (error) {
|
|
287
|
+
if (!(error instanceof ConversionError))
|
|
288
|
+
throw error;
|
|
289
|
+
unsupported.push({ file: source.name, reason: error.message });
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
for (const collector of [otlp, langfuse]) {
|
|
294
|
+
const finished = collector.finish();
|
|
295
|
+
for (const problem of finished.refused)
|
|
296
|
+
refused.push({
|
|
297
|
+
...(problem.traceId ? { trace_id: problem.traceId } : {}),
|
|
298
|
+
reason: problem.reason,
|
|
299
|
+
});
|
|
300
|
+
for (const trace of finished.traces) {
|
|
301
|
+
const first = [...trace.sourceDigests][0] ?? "";
|
|
302
|
+
merge(whole, conflicts, trace, first);
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
for (const [traceId, reason] of conflicts) {
|
|
306
|
+
whole.delete(traceId);
|
|
307
|
+
refused.push({ trace_id: traceId, reason });
|
|
308
|
+
}
|
|
309
|
+
const traces = [...whole.values()].sort((a, b) => (a.traceId < b.traceId ? -1 : 1));
|
|
310
|
+
if (!traces.length) {
|
|
311
|
+
const problems = [...unsupported, ...refused].slice(0, 3).map((item) => item.reason);
|
|
312
|
+
throw new PrepareError(`No trace was recognized in ${input}${problems.length ? ` (${problems.join("; ")})` : ""}`);
|
|
313
|
+
}
|
|
314
|
+
if (traces.length > traceImportLimits.traces)
|
|
315
|
+
throw new PrepareError(`The input holds ${traces.length} traces; one import takes at most ${traceImportLimits.traces}`);
|
|
316
|
+
await prepareOutputDirectory(outputDirectory);
|
|
317
|
+
const manifestTraces = [];
|
|
318
|
+
const formats = new Set();
|
|
319
|
+
const formatCounts = {
|
|
320
|
+
otlp_json: 0,
|
|
321
|
+
langfuse_observations: 0,
|
|
322
|
+
atif: 0,
|
|
323
|
+
};
|
|
324
|
+
const byOrigin = {
|
|
325
|
+
production: 0,
|
|
326
|
+
evaluation: 0,
|
|
327
|
+
simulation: 0,
|
|
328
|
+
unknown: 0,
|
|
329
|
+
};
|
|
330
|
+
const written = new Set();
|
|
331
|
+
let withoutIdentity = 0;
|
|
332
|
+
let remappedTraces = 0;
|
|
333
|
+
let remappedSpans = 0;
|
|
334
|
+
let blobCount = 0;
|
|
335
|
+
for (const trace of traces) {
|
|
336
|
+
const problem = trace.entries.length > traceImportLimits.spansPerTrace
|
|
337
|
+
? `more than ${traceImportLimits.spansPerTrace} spans`
|
|
338
|
+
: trace.blobs.length > traceImportLimits.blobsPerTrace
|
|
339
|
+
? `more than ${traceImportLimits.blobsPerTrace} large values`
|
|
340
|
+
: undefined;
|
|
341
|
+
if (problem) {
|
|
342
|
+
refused.push({ trace_id: trace.traceId, reason: problem });
|
|
343
|
+
continue;
|
|
344
|
+
}
|
|
345
|
+
let batches;
|
|
346
|
+
try {
|
|
347
|
+
batches = batchEntries(trace.entries);
|
|
348
|
+
}
|
|
349
|
+
catch (error) {
|
|
350
|
+
if (!(error instanceof ConversionError))
|
|
351
|
+
throw error;
|
|
352
|
+
refused.push({ trace_id: trace.traceId, reason: error.message });
|
|
353
|
+
continue;
|
|
354
|
+
}
|
|
355
|
+
if (batches.length > traceImportLimits.batchesPerTrace) {
|
|
356
|
+
refused.push({
|
|
357
|
+
trace_id: trace.traceId,
|
|
358
|
+
reason: `more than ${traceImportLimits.batchesPerTrace} batches`,
|
|
359
|
+
});
|
|
360
|
+
continue;
|
|
361
|
+
}
|
|
362
|
+
for (const batch of batches) {
|
|
363
|
+
const name = `${trace.traceId}.${batch.sha256}.json.gz`;
|
|
364
|
+
if (!written.has(name))
|
|
365
|
+
await writePrivate(join(outputDirectory, name), batch.body);
|
|
366
|
+
written.add(name);
|
|
367
|
+
}
|
|
368
|
+
for (const blob of trace.blobs) {
|
|
369
|
+
const name = `${blob.sha256}.blob`;
|
|
370
|
+
if (!written.has(name))
|
|
371
|
+
await writePrivate(join(outputDirectory, name), blob.bytes);
|
|
372
|
+
written.add(name);
|
|
373
|
+
}
|
|
374
|
+
const sourceDigest = trace.sourceDigests.size === 1 ? [...trace.sourceDigests][0] : undefined;
|
|
375
|
+
const declared = {
|
|
376
|
+
trace_id: trace.traceId,
|
|
377
|
+
span_count: trace.entries.length,
|
|
378
|
+
batches: batches.map((batch) => ({
|
|
379
|
+
sha256: batch.sha256,
|
|
380
|
+
bytes: batch.body.byteLength,
|
|
381
|
+
span_count: batch.spanIds.length,
|
|
382
|
+
})),
|
|
383
|
+
blobs: trace.blobs.map((blob) => ({
|
|
384
|
+
sha256: blob.sha256,
|
|
385
|
+
byte_size: blob.byteSize,
|
|
386
|
+
content_type: blob.contentType,
|
|
387
|
+
})),
|
|
388
|
+
...(trace.identity ? { identity: trace.identity } : {}),
|
|
389
|
+
origin: trace.origin,
|
|
390
|
+
...(sourceDigest ? { source_digest: sourceDigest } : {}),
|
|
391
|
+
};
|
|
392
|
+
manifestTraces.push(declared);
|
|
393
|
+
formats.add(trace.format);
|
|
394
|
+
formatCounts[trace.format]++;
|
|
395
|
+
byOrigin[trace.origin]++;
|
|
396
|
+
if (!manifestTraceHasIdentity(declared))
|
|
397
|
+
withoutIdentity++;
|
|
398
|
+
if (trace.remappedTraceId || trace.remappedSpans)
|
|
399
|
+
remappedTraces++;
|
|
400
|
+
remappedSpans += trace.remappedSpans;
|
|
401
|
+
blobCount += trace.blobs.length;
|
|
402
|
+
if (trace.format !== "atif")
|
|
403
|
+
for (const digest of trace.sourceDigests)
|
|
404
|
+
for (const name of fileNames.get(digest) ?? [])
|
|
405
|
+
tracesPerFile.set(name, (tracesPerFile.get(name) ?? 0) + 1);
|
|
406
|
+
}
|
|
407
|
+
if (!manifestTraces.length)
|
|
408
|
+
throw new PrepareError(`No trace of ${input} could be prepared: ${refused[0]?.reason ?? ""}`);
|
|
409
|
+
for (const entry of supported)
|
|
410
|
+
if (entry.format !== "atif")
|
|
411
|
+
entry.traces = tracesPerFile.get(entry.file) ?? 0;
|
|
412
|
+
const manifest = {
|
|
413
|
+
version: 1,
|
|
414
|
+
source: {
|
|
415
|
+
formats: traceImportFormats.filter((format) => formats.has(format)),
|
|
416
|
+
helper: "hue-sdk",
|
|
417
|
+
helper_version: sdkVersion,
|
|
418
|
+
},
|
|
419
|
+
traces: manifestTraces,
|
|
420
|
+
};
|
|
421
|
+
const invalid = validateManifest(manifest);
|
|
422
|
+
if (invalid)
|
|
423
|
+
throw new PrepareError(`The prepared manifest is not valid: ${invalid}`);
|
|
424
|
+
const bytes = Buffer.from(JSON.stringify(manifest), "utf8");
|
|
425
|
+
if (bytes.byteLength > traceImportLimits.manifestBytes)
|
|
426
|
+
throw new PrepareError("The manifest exceeds 8 MiB; prepare the input in smaller parts");
|
|
427
|
+
await writePrivate(join(outputDirectory, MANIFEST_FILE), bytes);
|
|
428
|
+
return {
|
|
429
|
+
manifest_digest: createHash("sha256").update(bytes).digest("hex"),
|
|
430
|
+
totals: manifestTotals(manifest),
|
|
431
|
+
source: manifest.source,
|
|
432
|
+
output_directory: outputDirectory,
|
|
433
|
+
formats: formatCounts,
|
|
434
|
+
files: { supported, unsupported },
|
|
435
|
+
traces: {
|
|
436
|
+
without_identity: withoutIdentity,
|
|
437
|
+
by_origin: byOrigin,
|
|
438
|
+
remapped: { traces: remappedTraces, spans: remappedSpans },
|
|
439
|
+
refused,
|
|
440
|
+
},
|
|
441
|
+
blobs: blobCount,
|
|
442
|
+
};
|
|
443
|
+
}
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
import { type TraceImportTotals } from "./traces-import-convert.js";
|
|
2
|
+
/**
|
|
3
|
+
* `hue traces import upload`: delivers a prepared bundle to Hue's import routes with the
|
|
4
|
+
* short-lived import authorization `prepare_trace_import` minted. The manifest's exact bytes
|
|
5
|
+
* are registered first; then, trace by trace in declared order and two requests at a time
|
|
6
|
+
* across traces, each large value is reserved, sent to the presigned URL and completed, each
|
|
7
|
+
* batch is posted with its digest, and the trace's receipt is read for every span id. A trace's
|
|
8
|
+
* requests run strictly in order. Transient answers and network failures are retried with
|
|
9
|
+
* bounded backoff (identical bytes are safe to resend), a rejected record is a conflict that is
|
|
10
|
+
* recorded rather than retried, and an authorization Hue no longer accepts stops the upload
|
|
11
|
+
* with the step to take. Progress is checkpointed to `progress.json` so a rerun resumes.
|
|
12
|
+
*/
|
|
13
|
+
export declare const UPLOAD_CONCURRENCY = 2;
|
|
14
|
+
export declare const RECEIPT_IDS_PER_CALL = 100;
|
|
15
|
+
export interface UploadIo {
|
|
16
|
+
/** The HTTP seam; `globalThis.fetch` by default. */
|
|
17
|
+
fetch?: (input: string, init: RequestInit) => Promise<Response>;
|
|
18
|
+
/** Waits `ms`, resolving early when `signal` aborts. */
|
|
19
|
+
sleep?: (ms: number, signal: AbortSignal) => Promise<void>;
|
|
20
|
+
/** Registers the stop request (SIGINT and SIGTERM by default) and returns its removal. */
|
|
21
|
+
onInterrupt?: (listener: () => void) => () => void;
|
|
22
|
+
/** Progress lines: text, or one JSON object per line with `--json`. */
|
|
23
|
+
progress?: (event: Record<string, unknown>) => void;
|
|
24
|
+
/** How long checkpoint writes are coalesced; 0 writes after every step. */
|
|
25
|
+
checkpointIntervalMs?: number;
|
|
26
|
+
}
|
|
27
|
+
export interface UploadOptions {
|
|
28
|
+
directory: string;
|
|
29
|
+
token: string;
|
|
30
|
+
importUrl: string;
|
|
31
|
+
}
|
|
32
|
+
/** Why the upload stopped before it completed. */
|
|
33
|
+
export declare class UploadStop extends Error {
|
|
34
|
+
readonly code: string;
|
|
35
|
+
/** Whether `prepare_trace_import` must be called again before the next run. */
|
|
36
|
+
readonly prepareAgain: boolean;
|
|
37
|
+
constructor(message: string, code: string,
|
|
38
|
+
/** Whether `prepare_trace_import` must be called again before the next run. */
|
|
39
|
+
prepareAgain?: boolean);
|
|
40
|
+
}
|
|
41
|
+
export interface UploadReport {
|
|
42
|
+
state: "completed" | "incomplete" | "stopped" | "failed";
|
|
43
|
+
import_url: string;
|
|
44
|
+
manifest_digest: string;
|
|
45
|
+
totals: TraceImportTotals;
|
|
46
|
+
traces: {
|
|
47
|
+
declared: number;
|
|
48
|
+
verified: number;
|
|
49
|
+
conflicted: number;
|
|
50
|
+
unverified: number;
|
|
51
|
+
};
|
|
52
|
+
batches: {
|
|
53
|
+
declared: number;
|
|
54
|
+
landed: number;
|
|
55
|
+
conflicted: number;
|
|
56
|
+
pending: number;
|
|
57
|
+
};
|
|
58
|
+
blobs: {
|
|
59
|
+
declared: number;
|
|
60
|
+
stored: number;
|
|
61
|
+
pending: number;
|
|
62
|
+
};
|
|
63
|
+
/** Hue's own line on the import's progress, when completion was refused. */
|
|
64
|
+
note?: string;
|
|
65
|
+
/** The traces Hue still lists as pending or conflicting when completion was refused. */
|
|
66
|
+
pending?: unknown[];
|
|
67
|
+
/** A batch or value Hue refused with a fixed code, or whose records it rejected: recorded,
|
|
68
|
+
* never sent again, reported with the trace. */
|
|
69
|
+
conflicts: {
|
|
70
|
+
trace_id: string;
|
|
71
|
+
sha256: string;
|
|
72
|
+
message: string;
|
|
73
|
+
}[];
|
|
74
|
+
next_action: string;
|
|
75
|
+
error?: {
|
|
76
|
+
code: string;
|
|
77
|
+
message: string;
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
interface TraceProgress {
|
|
81
|
+
state: "verified" | "pending";
|
|
82
|
+
blobs: string[];
|
|
83
|
+
batches: string[];
|
|
84
|
+
conflicts: {
|
|
85
|
+
sha256: string;
|
|
86
|
+
message: string;
|
|
87
|
+
}[];
|
|
88
|
+
}
|
|
89
|
+
interface ProgressFile {
|
|
90
|
+
version: 1;
|
|
91
|
+
manifest_digest: string;
|
|
92
|
+
import_url: string;
|
|
93
|
+
traces: Record<string, TraceProgress>;
|
|
94
|
+
}
|
|
95
|
+
/** The import authorization file, in either shape `prepare_trace_import`'s answer is saved in:
|
|
96
|
+
* its JSON (`authorization`, and `upload_url` beside it), or the authorization alone on the first
|
|
97
|
+
* line, optionally with the upload URL on the second. Nothing in it is ever printed. */
|
|
98
|
+
export declare function parseAuthorizationFile(content: string): {
|
|
99
|
+
token: string;
|
|
100
|
+
importUrl?: string;
|
|
101
|
+
};
|
|
102
|
+
/** The upload URL as `prepare_trace_import` returned it, or why it is refused. */
|
|
103
|
+
export declare function parseImportUrl(value: string, origin?: string): URL;
|
|
104
|
+
/** Writes `progress.json` (mode 0600, through a temporary file and an atomic rename) after each
|
|
105
|
+
* verified step, coalescing writes that fall within the interval. */
|
|
106
|
+
export declare class Progress {
|
|
107
|
+
private readonly path;
|
|
108
|
+
readonly file: ProgressFile;
|
|
109
|
+
private readonly intervalMs;
|
|
110
|
+
private dirty;
|
|
111
|
+
private timer;
|
|
112
|
+
private writing;
|
|
113
|
+
private lastWrite;
|
|
114
|
+
/** The first write that failed (the disk full, the directory gone or no longer writable);
|
|
115
|
+
* nothing is written after it, and the upload reports it instead of crashing. */
|
|
116
|
+
failure: NodeJS.ErrnoException | undefined;
|
|
117
|
+
/** Called once, when a write first fails. */
|
|
118
|
+
onFailure: (() => void) | undefined;
|
|
119
|
+
constructor(path: string, file: ProgressFile, intervalMs: number);
|
|
120
|
+
trace(traceId: string): TraceProgress;
|
|
121
|
+
/** Records a step; the file is written now, or at the end of the interval. Never rejects: a
|
|
122
|
+
* write that fails is kept in {@link failure}, whether it was awaited or ran from the timer. */
|
|
123
|
+
changed(): Promise<void>;
|
|
124
|
+
/** Writes what changed, now. Never rejects; see {@link failure}. */
|
|
125
|
+
flush(): Promise<void>;
|
|
126
|
+
}
|
|
127
|
+
/** Runs the upload; resolves to the report. Throws only for a programming error. */
|
|
128
|
+
export declare function uploadTraceBundle(options: UploadOptions, io?: UploadIo): Promise<UploadReport>;
|
|
129
|
+
export {};
|