deepline 0.1.300 → 0.1.301
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-store-adapter.ts +103 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-writer.ts +78 -5
- package/dist/cli/index.js +1 -1
- package/dist/cli/index.mjs +1 -1
- package/dist/index.js +1 -1
- package/dist/index.mjs +1 -1
- package/package.json +1 -1
|
@@ -155,7 +155,7 @@ export const SDK_RELEASE = {
|
|
|
155
155
|
// 0.1.253 makes play-page browser opening opt-in and retires --no-open.
|
|
156
156
|
// 0.1.254 removes the internal operations tree from the published SDK CLI.
|
|
157
157
|
// Operators use the checkout-local deepline-admin binary instead.
|
|
158
|
-
version: '0.1.
|
|
158
|
+
version: '0.1.301',
|
|
159
159
|
contracts: {
|
|
160
160
|
api: {
|
|
161
161
|
name: 'sdk-http-api',
|
|
@@ -124,6 +124,7 @@ export class RuntimeReceiptStoreAdapter {
|
|
|
124
124
|
maxBatchBytes?: number;
|
|
125
125
|
maxFlushMs?: number;
|
|
126
126
|
maxBufferedBytes?: number;
|
|
127
|
+
onRetryTelemetry?: (line: string) => void;
|
|
127
128
|
}) {
|
|
128
129
|
const context: WorkerRuntimeApiContext = {
|
|
129
130
|
...options.context,
|
|
@@ -137,6 +138,14 @@ export class RuntimeReceiptStoreAdapter {
|
|
|
137
138
|
maxFlushMs: options.maxFlushMs,
|
|
138
139
|
maxBufferedBytes: options.maxBufferedBytes,
|
|
139
140
|
classifyRetryableError: classifyReceiptRetry,
|
|
141
|
+
onRetryEvent: (event) =>
|
|
142
|
+
emitReceiptWriterRetryTelemetry({
|
|
143
|
+
event,
|
|
144
|
+
playName: options.playName,
|
|
145
|
+
runId: options.runId,
|
|
146
|
+
gatewayHost: runtimeGatewayHost(context.baseUrl),
|
|
147
|
+
onTelemetry: options.onRetryTelemetry,
|
|
148
|
+
}),
|
|
140
149
|
send: async (commands, { signal }) =>
|
|
141
150
|
await sendReceiptCommands({
|
|
142
151
|
context: { ...context, signal },
|
|
@@ -260,6 +269,100 @@ export class RuntimeReceiptStoreAdapter {
|
|
|
260
269
|
}
|
|
261
270
|
}
|
|
262
271
|
|
|
272
|
+
const RECEIPT_WRITER_RETRY_TELEMETRY_TAG =
|
|
273
|
+
'[perf][worker.receipt_writer.retry]';
|
|
274
|
+
|
|
275
|
+
function isPowerOfTwo(value: number): boolean {
|
|
276
|
+
return value > 0 && (value & (value - 1)) === 0;
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
function runtimeGatewayHost(baseUrl: string): string | null {
|
|
280
|
+
try {
|
|
281
|
+
return new URL(baseUrl).host || null;
|
|
282
|
+
} catch {
|
|
283
|
+
return null;
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
function receiptCommandTelemetryAction(command: ReceiptCommand): string {
|
|
288
|
+
return command.kind === 'settle'
|
|
289
|
+
? `${command.kind}_${command.outcome}`
|
|
290
|
+
: command.kind;
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
function receiptRetryErrorTelemetry(error: unknown): Record<string, unknown> {
|
|
294
|
+
if (isAppRuntimeApiCapacityError(error)) {
|
|
295
|
+
return {
|
|
296
|
+
failureKind: 'capacity',
|
|
297
|
+
errorName: error.name,
|
|
298
|
+
status: error.status,
|
|
299
|
+
code: error.code,
|
|
300
|
+
requestId: error.requestId,
|
|
301
|
+
};
|
|
302
|
+
}
|
|
303
|
+
if (error instanceof AppRuntimeApiTransportError) {
|
|
304
|
+
return {
|
|
305
|
+
failureKind: 'transport',
|
|
306
|
+
errorName: error.name,
|
|
307
|
+
transportAttempts: error.attempts,
|
|
308
|
+
transportAction: error.action,
|
|
309
|
+
};
|
|
310
|
+
}
|
|
311
|
+
if (error instanceof AppRuntimeApiResponseError) {
|
|
312
|
+
return {
|
|
313
|
+
failureKind: 'http',
|
|
314
|
+
errorName: error.name,
|
|
315
|
+
status: error.status,
|
|
316
|
+
code: error.code,
|
|
317
|
+
requestId: error.requestId,
|
|
318
|
+
};
|
|
319
|
+
}
|
|
320
|
+
return {
|
|
321
|
+
failureKind: 'unknown',
|
|
322
|
+
errorName: error instanceof Error ? error.name : typeof error,
|
|
323
|
+
};
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
function emitReceiptWriterRetryTelemetry(input: {
|
|
327
|
+
event: import('./runtime-receipt-writer').RuntimeReceiptWriterRetryEvent<ReceiptCommand>;
|
|
328
|
+
playName: string;
|
|
329
|
+
runId: string;
|
|
330
|
+
gatewayHost: string | null;
|
|
331
|
+
onTelemetry?: (line: string) => void;
|
|
332
|
+
}): void {
|
|
333
|
+
const { event } = input;
|
|
334
|
+
if (event.phase === 'retry' && !isPowerOfTwo(event.attempt)) return;
|
|
335
|
+
const payload = {
|
|
336
|
+
phase: event.phase,
|
|
337
|
+
playName: input.playName,
|
|
338
|
+
runId: input.runId,
|
|
339
|
+
gatewayHost: input.gatewayHost,
|
|
340
|
+
action: receiptCommandTelemetryAction(event.firstInput),
|
|
341
|
+
batchId: event.batchId,
|
|
342
|
+
batchSize: event.batchSize,
|
|
343
|
+
attempt: event.attempt,
|
|
344
|
+
elapsedMs: event.elapsedMs,
|
|
345
|
+
retryAfterMs: event.retryAfterMs,
|
|
346
|
+
queued: event.queued,
|
|
347
|
+
blocked: event.blocked,
|
|
348
|
+
bufferedBytes: event.bufferedBytes,
|
|
349
|
+
...(event.error === undefined
|
|
350
|
+
? { failureKind: null }
|
|
351
|
+
: receiptRetryErrorTelemetry(event.error)),
|
|
352
|
+
};
|
|
353
|
+
try {
|
|
354
|
+
const serialized = JSON.stringify(payload);
|
|
355
|
+
input.onTelemetry?.(`${RECEIPT_WRITER_RETRY_TELEMETRY_TAG} ${serialized}`);
|
|
356
|
+
if (event.phase === 'retry') {
|
|
357
|
+
console.warn(RECEIPT_WRITER_RETRY_TELEMETRY_TAG, serialized);
|
|
358
|
+
} else {
|
|
359
|
+
console.info(RECEIPT_WRITER_RETRY_TELEMETRY_TAG, serialized);
|
|
360
|
+
}
|
|
361
|
+
} catch {
|
|
362
|
+
// Receipt telemetry must never affect durable delivery.
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
|
|
263
366
|
function receiptCommandBatchKey(command: ReceiptCommand): string {
|
|
264
367
|
switch (command.kind) {
|
|
265
368
|
case 'get':
|
|
@@ -22,6 +22,20 @@ export type RuntimeReceiptWriterStats = {
|
|
|
22
22
|
bufferedBytes: number;
|
|
23
23
|
};
|
|
24
24
|
|
|
25
|
+
export type RuntimeReceiptWriterRetryEvent<Input> = {
|
|
26
|
+
phase: 'retry' | 'recovered';
|
|
27
|
+
attempt: number;
|
|
28
|
+
batchId: number;
|
|
29
|
+
batchSize: number;
|
|
30
|
+
firstInput: Input;
|
|
31
|
+
elapsedMs: number;
|
|
32
|
+
retryAfterMs: number;
|
|
33
|
+
queued: number;
|
|
34
|
+
blocked: number;
|
|
35
|
+
bufferedBytes: number;
|
|
36
|
+
error?: unknown;
|
|
37
|
+
};
|
|
38
|
+
|
|
25
39
|
export class RuntimeReceiptWriterBufferLimitError extends Error {
|
|
26
40
|
constructor(
|
|
27
41
|
readonly inputBytes: number,
|
|
@@ -76,11 +90,15 @@ export class RuntimeReceiptWriter<Input, Output> {
|
|
|
76
90
|
readonly #maxBatchBytes: number;
|
|
77
91
|
readonly #maxBufferedBytes: number;
|
|
78
92
|
readonly #maxFlushMs: number;
|
|
93
|
+
readonly #onRetryEvent:
|
|
94
|
+
| ((event: RuntimeReceiptWriterRetryEvent<Input>) => void)
|
|
95
|
+
| null;
|
|
79
96
|
readonly #queued: Array<PendingWrite<Input, Output>> = [];
|
|
80
97
|
readonly #blocked: Array<PendingWrite<Input, Output>> = [];
|
|
81
98
|
readonly #closeController = new AbortController();
|
|
82
99
|
|
|
83
100
|
#bufferedBytes = 0;
|
|
101
|
+
#nextBatchId = 0;
|
|
84
102
|
#activeBatch: Array<PendingWrite<Input, Output>> | null = null;
|
|
85
103
|
#pump: Promise<void> | null = null;
|
|
86
104
|
#flushTimer: ReturnType<typeof setTimeout> | null = null;
|
|
@@ -103,6 +121,7 @@ export class RuntimeReceiptWriter<Input, Output> {
|
|
|
103
121
|
maxBatchBytes?: number;
|
|
104
122
|
maxBufferedBytes?: number;
|
|
105
123
|
maxFlushMs?: number;
|
|
124
|
+
onRetryEvent?: (event: RuntimeReceiptWriterRetryEvent<Input>) => void;
|
|
106
125
|
}) {
|
|
107
126
|
this.#batchKey = options.batchKey;
|
|
108
127
|
this.#send = options.send;
|
|
@@ -119,6 +138,7 @@ export class RuntimeReceiptWriter<Input, Output> {
|
|
|
119
138
|
this.#maxBufferedBytes,
|
|
120
139
|
);
|
|
121
140
|
this.#maxFlushMs = Math.max(0, Math.floor(options.maxFlushMs ?? 5));
|
|
141
|
+
this.#onRetryEvent = options.onRetryEvent ?? null;
|
|
122
142
|
}
|
|
123
143
|
|
|
124
144
|
write(input: Input, options: { signal?: AbortSignal } = {}): Promise<Output> {
|
|
@@ -243,9 +263,10 @@ export class RuntimeReceiptWriter<Input, Output> {
|
|
|
243
263
|
while (this.#queued.length > 0) {
|
|
244
264
|
const batch = this.#takeBatch();
|
|
245
265
|
if (batch.length === 0) continue;
|
|
266
|
+
const batchId = ++this.#nextBatchId;
|
|
246
267
|
this.#activeBatch = batch;
|
|
247
268
|
try {
|
|
248
|
-
const outputs = await this.#sendWithRetry(batch);
|
|
269
|
+
const outputs = await this.#sendWithRetry(batch, batchId);
|
|
249
270
|
if (outputs.length !== batch.length) {
|
|
250
271
|
throw new RuntimeReceiptWriterResultCountError(
|
|
251
272
|
batch.length,
|
|
@@ -305,9 +326,13 @@ export class RuntimeReceiptWriter<Input, Output> {
|
|
|
305
326
|
|
|
306
327
|
async #sendWithRetry(
|
|
307
328
|
batch: readonly PendingWrite<Input, Output>[],
|
|
329
|
+
batchId: number,
|
|
308
330
|
): Promise<readonly Output[]> {
|
|
309
331
|
const inputs = batch.map((pending) => pending.input);
|
|
332
|
+
const startedAt = Date.now();
|
|
333
|
+
let attempt = 0;
|
|
310
334
|
while (true) {
|
|
335
|
+
attempt += 1;
|
|
311
336
|
if (this.#closeController.signal.aborted) {
|
|
312
337
|
throw this.#closeReason;
|
|
313
338
|
}
|
|
@@ -315,20 +340,68 @@ export class RuntimeReceiptWriter<Input, Output> {
|
|
|
315
340
|
throw batch[0]?.aborted;
|
|
316
341
|
}
|
|
317
342
|
try {
|
|
318
|
-
|
|
343
|
+
const outputs = await this.#send(inputs, {
|
|
319
344
|
signal: this.#closeController.signal,
|
|
320
345
|
});
|
|
346
|
+
if (attempt > 1) {
|
|
347
|
+
this.#emitRetryEvent({
|
|
348
|
+
phase: 'recovered',
|
|
349
|
+
attempt,
|
|
350
|
+
batchId,
|
|
351
|
+
batch,
|
|
352
|
+
startedAt,
|
|
353
|
+
retryAfterMs: 0,
|
|
354
|
+
});
|
|
355
|
+
}
|
|
356
|
+
return outputs;
|
|
321
357
|
} catch (error) {
|
|
322
358
|
const retry = this.#classifyRetryableError(error);
|
|
323
359
|
if (!retry) throw error;
|
|
324
|
-
|
|
325
|
-
|
|
360
|
+
const retryAfterMs = jitteredRetryDelayMs(retry.retryAfterMs);
|
|
361
|
+
this.#emitRetryEvent({
|
|
362
|
+
phase: 'retry',
|
|
363
|
+
attempt,
|
|
364
|
+
batchId,
|
|
326
365
|
batch,
|
|
327
|
-
|
|
366
|
+
startedAt,
|
|
367
|
+
retryAfterMs,
|
|
368
|
+
error,
|
|
369
|
+
});
|
|
370
|
+
await this.#waitForRetry(retryAfterMs, batch);
|
|
328
371
|
}
|
|
329
372
|
}
|
|
330
373
|
}
|
|
331
374
|
|
|
375
|
+
#emitRetryEvent(input: {
|
|
376
|
+
phase: 'retry' | 'recovered';
|
|
377
|
+
attempt: number;
|
|
378
|
+
batchId: number;
|
|
379
|
+
batch: readonly PendingWrite<Input, Output>[];
|
|
380
|
+
startedAt: number;
|
|
381
|
+
retryAfterMs: number;
|
|
382
|
+
error?: unknown;
|
|
383
|
+
}): void {
|
|
384
|
+
const firstInput = input.batch[0]?.input;
|
|
385
|
+
if (!this.#onRetryEvent || firstInput === undefined) return;
|
|
386
|
+
try {
|
|
387
|
+
this.#onRetryEvent({
|
|
388
|
+
phase: input.phase,
|
|
389
|
+
attempt: input.attempt,
|
|
390
|
+
batchId: input.batchId,
|
|
391
|
+
batchSize: input.batch.length,
|
|
392
|
+
firstInput,
|
|
393
|
+
elapsedMs: Date.now() - input.startedAt,
|
|
394
|
+
retryAfterMs: input.retryAfterMs,
|
|
395
|
+
queued: this.#queued.length,
|
|
396
|
+
blocked: this.#blocked.length,
|
|
397
|
+
bufferedBytes: this.#bufferedBytes,
|
|
398
|
+
...(input.error === undefined ? {} : { error: input.error }),
|
|
399
|
+
});
|
|
400
|
+
} catch {
|
|
401
|
+
// Receipt telemetry must never affect durable delivery.
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
|
|
332
405
|
async #waitForRetry(
|
|
333
406
|
delayMs: number,
|
|
334
407
|
batch: readonly PendingWrite<Input, Output>[],
|
package/dist/cli/index.js
CHANGED
|
@@ -718,7 +718,7 @@ var SDK_RELEASE = {
|
|
|
718
718
|
// 0.1.253 makes play-page browser opening opt-in and retires --no-open.
|
|
719
719
|
// 0.1.254 removes the internal operations tree from the published SDK CLI.
|
|
720
720
|
// Operators use the checkout-local deepline-admin binary instead.
|
|
721
|
-
version: "0.1.
|
|
721
|
+
version: "0.1.301",
|
|
722
722
|
contracts: {
|
|
723
723
|
api: {
|
|
724
724
|
name: "sdk-http-api",
|
package/dist/cli/index.mjs
CHANGED
|
@@ -703,7 +703,7 @@ var SDK_RELEASE = {
|
|
|
703
703
|
// 0.1.253 makes play-page browser opening opt-in and retires --no-open.
|
|
704
704
|
// 0.1.254 removes the internal operations tree from the published SDK CLI.
|
|
705
705
|
// Operators use the checkout-local deepline-admin binary instead.
|
|
706
|
-
version: "0.1.
|
|
706
|
+
version: "0.1.301",
|
|
707
707
|
contracts: {
|
|
708
708
|
api: {
|
|
709
709
|
name: "sdk-http-api",
|
package/dist/index.js
CHANGED
|
@@ -438,7 +438,7 @@ var SDK_RELEASE = {
|
|
|
438
438
|
// 0.1.253 makes play-page browser opening opt-in and retires --no-open.
|
|
439
439
|
// 0.1.254 removes the internal operations tree from the published SDK CLI.
|
|
440
440
|
// Operators use the checkout-local deepline-admin binary instead.
|
|
441
|
-
version: "0.1.
|
|
441
|
+
version: "0.1.301",
|
|
442
442
|
contracts: {
|
|
443
443
|
api: {
|
|
444
444
|
name: "sdk-http-api",
|
package/dist/index.mjs
CHANGED
|
@@ -367,7 +367,7 @@ var SDK_RELEASE = {
|
|
|
367
367
|
// 0.1.253 makes play-page browser opening opt-in and retires --no-open.
|
|
368
368
|
// 0.1.254 removes the internal operations tree from the published SDK CLI.
|
|
369
369
|
// Operators use the checkout-local deepline-admin binary instead.
|
|
370
|
-
version: "0.1.
|
|
370
|
+
version: "0.1.301",
|
|
371
371
|
contracts: {
|
|
372
372
|
api: {
|
|
373
373
|
name: "sdk-http-api",
|