deepline 0.1.91 → 0.1.94
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/index.js +4012 -705
- package/dist/cli/index.mjs +4028 -714
- package/dist/index.d.mts +232 -108
- package/dist/index.d.ts +232 -108
- package/dist/index.js +1145 -99
- package/dist/index.mjs +1134 -99
- package/dist/repo/apps/play-runner-workers/src/coordinator-entry.ts +87 -20
- package/dist/repo/apps/play-runner-workers/src/entry.ts +75 -22
- package/dist/repo/sdk/src/client.ts +412 -40
- package/dist/repo/sdk/src/index.ts +1 -0
- package/dist/repo/sdk/src/play.ts +51 -0
- package/dist/repo/sdk/src/release.ts +2 -2
- package/dist/repo/sdk/src/runs/observe-transport.ts +481 -0
- package/dist/repo/sdk/src/stream-reconnect.ts +44 -0
- package/dist/repo/sdk/src/types.ts +10 -3
- package/dist/repo/shared_libs/play-runtime/email-status.ts +10 -36
- package/dist/repo/shared_libs/play-runtime/extractor-targets.ts +3 -3
- package/dist/repo/shared_libs/play-runtime/live-events.ts +217 -0
- package/dist/repo/shared_libs/play-runtime/run-ledger.ts +1074 -0
- package/dist/repo/shared_libs/play-runtime/run-snapshot-stream.ts +581 -0
- package/dist/repo/shared_libs/play-runtime/tool-result.ts +44 -0
- package/dist/repo/shared_libs/plays/secret-guardrails.ts +22 -11
- package/package.json +5 -2
|
@@ -0,0 +1,581 @@
|
|
|
1
|
+
import { resolveTimingWindow } from './live-events';
|
|
2
|
+
import {
|
|
3
|
+
normalizePlayRunLedgerSnapshot,
|
|
4
|
+
type PlayRunLedgerSnapshot,
|
|
5
|
+
type PlayRunLedgerStepSnapshot,
|
|
6
|
+
} from './run-ledger';
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Run Snapshot Stream.
|
|
10
|
+
*
|
|
11
|
+
* The canonical client-side projection of a persisted Play Run document into
|
|
12
|
+
* the live snapshot shape, plus the differ that turns successive snapshots
|
|
13
|
+
* into incremental `play.*` live events. This module is shared between the
|
|
14
|
+
* Vercel app (legacy SSE shim, dashboard) and the SDK subscription transport
|
|
15
|
+
* (`sdk/src/runs/observe-transport.ts`), so both watchers render identical
|
|
16
|
+
* event streams from the same Convex Run Snapshot. See ADR-0008.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
export type PlayRunLiveStatus =
|
|
20
|
+
| 'queued'
|
|
21
|
+
| 'running'
|
|
22
|
+
| 'completed'
|
|
23
|
+
| 'failed'
|
|
24
|
+
| 'cancelled'
|
|
25
|
+
| 'terminated'
|
|
26
|
+
| 'timed_out'
|
|
27
|
+
| 'unknown';
|
|
28
|
+
|
|
29
|
+
export type PlayRunStreamNodeProgress = {
|
|
30
|
+
completed?: number;
|
|
31
|
+
total?: number;
|
|
32
|
+
failed?: number;
|
|
33
|
+
message?: string;
|
|
34
|
+
updatedAt?: number | null;
|
|
35
|
+
startedAt?: number | null;
|
|
36
|
+
completedAt?: number | null;
|
|
37
|
+
artifactTableNamespace?: string | null;
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
export type PlayRunStreamNodeState = {
|
|
41
|
+
nodeId: string;
|
|
42
|
+
status: 'idle' | 'running' | 'completed' | 'failed' | 'skipped';
|
|
43
|
+
artifactTableNamespace?: string | null;
|
|
44
|
+
progress?: PlayRunStreamNodeProgress | null;
|
|
45
|
+
startedAt?: number | null;
|
|
46
|
+
completedAt?: number | null;
|
|
47
|
+
updatedAt?: number | null;
|
|
48
|
+
};
|
|
49
|
+
|
|
50
|
+
export type PlayRunLiveSnapshot = {
|
|
51
|
+
runId: string;
|
|
52
|
+
status: PlayRunLiveStatus;
|
|
53
|
+
updatedAt: number | null;
|
|
54
|
+
/**
|
|
55
|
+
* Rotating log tail (the ledger snapshot's bounded `logTail`; the wire name
|
|
56
|
+
* stays `logs` for installed clients). `totalLogCount` is the run's
|
|
57
|
+
* cumulative ingested-line count (monotonic), so stream differs can cursor
|
|
58
|
+
* on absolute sequence numbers instead of indexes into the rotated tail.
|
|
59
|
+
* Full retention lives in the Run Log Stream (`GET /api/v2/runs/:id/logs`).
|
|
60
|
+
*/
|
|
61
|
+
logs: string[];
|
|
62
|
+
totalLogCount: number;
|
|
63
|
+
/**
|
|
64
|
+
* True once the Run Log Stream stopped storing log bodies because the run
|
|
65
|
+
* crossed the retention cap (ADR-0009). Additive/optional on the wire.
|
|
66
|
+
*/
|
|
67
|
+
logsTruncated?: boolean;
|
|
68
|
+
activeArtifactTableNamespace: string | null;
|
|
69
|
+
resultTableNamespace: string | null;
|
|
70
|
+
nodeStates: PlayRunStreamNodeState[];
|
|
71
|
+
activeNodeId: string | null;
|
|
72
|
+
};
|
|
73
|
+
|
|
74
|
+
export function normalizePlayRunLiveStatus(value: unknown): PlayRunLiveStatus {
|
|
75
|
+
const normalized = String(value ?? '')
|
|
76
|
+
.trim()
|
|
77
|
+
.toLowerCase();
|
|
78
|
+
switch (normalized) {
|
|
79
|
+
case 'queued':
|
|
80
|
+
case 'pending':
|
|
81
|
+
return 'running';
|
|
82
|
+
case 'running':
|
|
83
|
+
case 'started':
|
|
84
|
+
return 'running';
|
|
85
|
+
case 'completed':
|
|
86
|
+
case 'complete':
|
|
87
|
+
case 'succeeded':
|
|
88
|
+
return 'completed';
|
|
89
|
+
case 'failed':
|
|
90
|
+
case 'error':
|
|
91
|
+
return 'failed';
|
|
92
|
+
case 'cancelled':
|
|
93
|
+
case 'canceled':
|
|
94
|
+
return 'cancelled';
|
|
95
|
+
case 'terminated':
|
|
96
|
+
return 'terminated';
|
|
97
|
+
case 'timed_out':
|
|
98
|
+
case 'timeout':
|
|
99
|
+
return 'timed_out';
|
|
100
|
+
default:
|
|
101
|
+
return 'unknown';
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
export function isTerminalPlayRunLiveStatus(
|
|
106
|
+
status: PlayRunLiveStatus,
|
|
107
|
+
): boolean {
|
|
108
|
+
return (
|
|
109
|
+
status === 'completed' ||
|
|
110
|
+
status === 'failed' ||
|
|
111
|
+
status === 'cancelled' ||
|
|
112
|
+
status === 'terminated' ||
|
|
113
|
+
status === 'timed_out'
|
|
114
|
+
);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
export function isActivePlayRunStatus(status: unknown): boolean {
|
|
118
|
+
return normalizePlayRunLiveStatus(status) === 'running';
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* The minimal persisted Play Run document shape required to project the live
|
|
123
|
+
* snapshot. This is a structural subset of the Convex `playRuns` doc and of
|
|
124
|
+
* the run-observer projection returned by
|
|
125
|
+
* `convex/runObservers.getPlayRunSnapshotForObserver`.
|
|
126
|
+
*/
|
|
127
|
+
export type LedgerBackedRunLike = {
|
|
128
|
+
workflowId: string;
|
|
129
|
+
status: string;
|
|
130
|
+
name?: string | null;
|
|
131
|
+
createdAt?: number | null;
|
|
132
|
+
startedAt?: number | null;
|
|
133
|
+
finishedAt?: number | null;
|
|
134
|
+
updatedAt?: number | null;
|
|
135
|
+
runSnapshot?: unknown;
|
|
136
|
+
};
|
|
137
|
+
|
|
138
|
+
function buildSnapshotFromLedger(
|
|
139
|
+
snapshot: PlayRunLedgerSnapshot,
|
|
140
|
+
): PlayRunLiveSnapshot {
|
|
141
|
+
const nodeStates = snapshot.orderedStepIds
|
|
142
|
+
.map((stepId) => snapshot.stepsById[stepId])
|
|
143
|
+
.filter((step): step is PlayRunLedgerStepSnapshot => Boolean(step))
|
|
144
|
+
.map((step) => ({
|
|
145
|
+
nodeId: step.stepId,
|
|
146
|
+
status: step.status,
|
|
147
|
+
artifactTableNamespace: step.artifactTableNamespace ?? null,
|
|
148
|
+
progress: step.progress
|
|
149
|
+
? {
|
|
150
|
+
completed: step.progress.completed,
|
|
151
|
+
total: step.progress.total,
|
|
152
|
+
failed: step.progress.failed,
|
|
153
|
+
message: step.progress.message,
|
|
154
|
+
artifactTableNamespace:
|
|
155
|
+
step.progress.artifactTableNamespace ??
|
|
156
|
+
step.artifactTableNamespace ??
|
|
157
|
+
null,
|
|
158
|
+
startedAt: step.startedAt ?? null,
|
|
159
|
+
completedAt: step.completedAt ?? null,
|
|
160
|
+
updatedAt: step.progress.updatedAt ?? step.updatedAt ?? null,
|
|
161
|
+
}
|
|
162
|
+
: null,
|
|
163
|
+
startedAt: step.startedAt ?? null,
|
|
164
|
+
completedAt: step.completedAt ?? null,
|
|
165
|
+
updatedAt: step.updatedAt ?? null,
|
|
166
|
+
}));
|
|
167
|
+
return {
|
|
168
|
+
runId: snapshot.runId,
|
|
169
|
+
status: normalizePlayRunLiveStatus(snapshot.status),
|
|
170
|
+
updatedAt:
|
|
171
|
+
snapshot.updatedAt ?? snapshot.finishedAt ?? snapshot.startedAt ?? null,
|
|
172
|
+
logs: snapshot.logTail,
|
|
173
|
+
totalLogCount: snapshot.totalLogCount,
|
|
174
|
+
...(snapshot.logsTruncated ? { logsTruncated: true } : {}),
|
|
175
|
+
activeArtifactTableNamespace: snapshot.activeArtifactTableNamespace ?? null,
|
|
176
|
+
resultTableNamespace: snapshot.resultTableNamespace ?? null,
|
|
177
|
+
nodeStates,
|
|
178
|
+
activeNodeId: snapshot.activeStepId ?? null,
|
|
179
|
+
};
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
export function buildPlayRunStatusSnapshot(input: {
|
|
183
|
+
run: LedgerBackedRunLike;
|
|
184
|
+
}): PlayRunLiveSnapshot {
|
|
185
|
+
const ledgerSnapshot = normalizePlayRunLedgerSnapshot(input.run.runSnapshot, {
|
|
186
|
+
runId: input.run.workflowId,
|
|
187
|
+
playName: input.run.name ?? null,
|
|
188
|
+
status: input.run.status,
|
|
189
|
+
createdAt: input.run.createdAt ?? null,
|
|
190
|
+
startedAt: input.run.startedAt ?? null,
|
|
191
|
+
updatedAt: input.run.updatedAt ?? null,
|
|
192
|
+
finishedAt: input.run.finishedAt ?? null,
|
|
193
|
+
});
|
|
194
|
+
return buildSnapshotFromLedger(ledgerSnapshot);
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/** Generic live-event envelope shape shared with the SSE protocol. */
|
|
198
|
+
export type RunStreamEventEnvelope<TPayload, TType extends string> = {
|
|
199
|
+
cursor: string;
|
|
200
|
+
streamId: string;
|
|
201
|
+
scope: 'play';
|
|
202
|
+
type: TType;
|
|
203
|
+
at: string;
|
|
204
|
+
payload: TPayload;
|
|
205
|
+
};
|
|
206
|
+
|
|
207
|
+
export type PlayRunStreamEvent =
|
|
208
|
+
| RunStreamEventEnvelope<
|
|
209
|
+
{
|
|
210
|
+
runId: string;
|
|
211
|
+
status: PlayRunLiveStatus;
|
|
212
|
+
updatedAt: number | null;
|
|
213
|
+
},
|
|
214
|
+
'play.run.status'
|
|
215
|
+
>
|
|
216
|
+
| RunStreamEventEnvelope<PlayRunLiveSnapshot, 'play.run.snapshot'>
|
|
217
|
+
| RunStreamEventEnvelope<
|
|
218
|
+
{
|
|
219
|
+
runId: string;
|
|
220
|
+
stepId: string;
|
|
221
|
+
status: PlayRunStreamNodeState['status'];
|
|
222
|
+
artifactTableNamespace: string | null;
|
|
223
|
+
startedAt?: number | null;
|
|
224
|
+
completedAt?: number | null;
|
|
225
|
+
updatedAt?: number | null;
|
|
226
|
+
},
|
|
227
|
+
'play.step.status'
|
|
228
|
+
>
|
|
229
|
+
| RunStreamEventEnvelope<
|
|
230
|
+
{
|
|
231
|
+
runId: string;
|
|
232
|
+
stepId: string;
|
|
233
|
+
completed?: number;
|
|
234
|
+
total?: number;
|
|
235
|
+
failed?: number;
|
|
236
|
+
message?: string;
|
|
237
|
+
artifactTableNamespace: string | null;
|
|
238
|
+
startedAt?: number | null;
|
|
239
|
+
completedAt?: number | null;
|
|
240
|
+
updatedAt?: number | null;
|
|
241
|
+
},
|
|
242
|
+
'play.step.progress'
|
|
243
|
+
>
|
|
244
|
+
| RunStreamEventEnvelope<PlayRunLogStreamPayload, 'play.run.log'>;
|
|
245
|
+
|
|
246
|
+
/**
|
|
247
|
+
* Log lines for one `play.run.log` event. `firstSeq`/`totalLogCount` are
|
|
248
|
+
* additive (ADR-0009): when `firstSeq` is present, `lines` is a contiguous
|
|
249
|
+
* run of log lines whose absolute (1-based) sequences are
|
|
250
|
+
* `firstSeq .. firstSeq + lines.length - 1`, letting clients append by seq
|
|
251
|
+
* instead of text-deduping or replacing from snapshots. When absent (gap
|
|
252
|
+
* marker payloads), clients append the lines verbatim.
|
|
253
|
+
*/
|
|
254
|
+
export type PlayRunLogStreamPayload = {
|
|
255
|
+
runId: string;
|
|
256
|
+
lines: string[];
|
|
257
|
+
source: string;
|
|
258
|
+
firstSeq?: number;
|
|
259
|
+
totalLogCount?: number;
|
|
260
|
+
};
|
|
261
|
+
|
|
262
|
+
function makeRunStreamEvent<TPayload, TType extends string>(input: {
|
|
263
|
+
cursor: string;
|
|
264
|
+
streamId: string;
|
|
265
|
+
type: TType;
|
|
266
|
+
payload: TPayload;
|
|
267
|
+
at?: string;
|
|
268
|
+
}): RunStreamEventEnvelope<TPayload, TType> {
|
|
269
|
+
return {
|
|
270
|
+
...input,
|
|
271
|
+
scope: 'play',
|
|
272
|
+
at: input.at ?? new Date().toISOString(),
|
|
273
|
+
};
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
export type PlayRunStreamDiffState = {
|
|
277
|
+
runSignature: string;
|
|
278
|
+
snapshotSignature: string;
|
|
279
|
+
stepStatusSignature: string;
|
|
280
|
+
stepProgressSignature: string;
|
|
281
|
+
/**
|
|
282
|
+
* Absolute sequence number (1-based, monotonic per run) of the last log
|
|
283
|
+
* line emitted to this stream. Cursors on `snapshot.totalLogCount`, not on
|
|
284
|
+
* indexes into the rotated log tail.
|
|
285
|
+
*/
|
|
286
|
+
lastLogSeq: number;
|
|
287
|
+
};
|
|
288
|
+
|
|
289
|
+
export const EMPTY_PLAY_RUN_STREAM_DIFF_STATE: PlayRunStreamDiffState = {
|
|
290
|
+
runSignature: '',
|
|
291
|
+
snapshotSignature: '',
|
|
292
|
+
stepStatusSignature: '',
|
|
293
|
+
stepProgressSignature: '',
|
|
294
|
+
lastLogSeq: 0,
|
|
295
|
+
};
|
|
296
|
+
|
|
297
|
+
function getSnapshotCursor(snapshot: PlayRunLiveSnapshot): string {
|
|
298
|
+
return String(snapshot.updatedAt ?? Date.now());
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
function getRunSignature(snapshot: PlayRunLiveSnapshot): string {
|
|
302
|
+
return [snapshot.runId, snapshot.status, snapshot.updatedAt ?? 0].join(':');
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
function getStepStatusSignature(snapshot: PlayRunLiveSnapshot): string {
|
|
306
|
+
return snapshot.nodeStates
|
|
307
|
+
.map((state) =>
|
|
308
|
+
[
|
|
309
|
+
state.nodeId,
|
|
310
|
+
state.status,
|
|
311
|
+
state.artifactTableNamespace ?? '',
|
|
312
|
+
state.startedAt ?? '',
|
|
313
|
+
state.completedAt ?? '',
|
|
314
|
+
state.progress?.startedAt ?? '',
|
|
315
|
+
state.progress?.completedAt ?? '',
|
|
316
|
+
state.updatedAt ?? '',
|
|
317
|
+
state.progress?.updatedAt ?? '',
|
|
318
|
+
].join(':'),
|
|
319
|
+
)
|
|
320
|
+
.join('|');
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
function getStepProgressSignature(snapshot: PlayRunLiveSnapshot): string {
|
|
324
|
+
return snapshot.nodeStates
|
|
325
|
+
.map((state) =>
|
|
326
|
+
[
|
|
327
|
+
state.nodeId,
|
|
328
|
+
state.progress?.completed ?? '',
|
|
329
|
+
state.progress?.total ?? '',
|
|
330
|
+
state.progress?.failed ?? '',
|
|
331
|
+
state.progress?.artifactTableNamespace ?? '',
|
|
332
|
+
state.progress?.startedAt ?? '',
|
|
333
|
+
state.progress?.completedAt ?? '',
|
|
334
|
+
state.progress?.updatedAt ?? '',
|
|
335
|
+
state.progress?.message ?? '',
|
|
336
|
+
].join(':'),
|
|
337
|
+
)
|
|
338
|
+
.join('|');
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
function getSnapshotSignature(snapshot: PlayRunLiveSnapshot): string {
|
|
342
|
+
return JSON.stringify(snapshot);
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
export type PlayRunLogGap = {
|
|
346
|
+
/** Number of log lines that fell out of the retained tail window. */
|
|
347
|
+
missingCount: number;
|
|
348
|
+
/** Absolute (1-based) sequence of the first retained tail line. */
|
|
349
|
+
tailFirstSeq: number;
|
|
350
|
+
};
|
|
351
|
+
|
|
352
|
+
/**
|
|
353
|
+
* Resolve whether the differ would have to skip log lines because the cursor
|
|
354
|
+
* fell behind the retained tail window. Transports can use this to backfill
|
|
355
|
+
* the gap from the durable run-event ledger before running the differ.
|
|
356
|
+
*/
|
|
357
|
+
export function resolvePlayRunLogGap(
|
|
358
|
+
snapshot: PlayRunLiveSnapshot,
|
|
359
|
+
lastLogSeq: number,
|
|
360
|
+
): PlayRunLogGap | null {
|
|
361
|
+
if (snapshot.totalLogCount <= lastLogSeq) {
|
|
362
|
+
return null;
|
|
363
|
+
}
|
|
364
|
+
const tailFirstSeq = snapshot.totalLogCount - snapshot.logs.length + 1;
|
|
365
|
+
if (lastLogSeq + 1 >= tailFirstSeq) {
|
|
366
|
+
return null;
|
|
367
|
+
}
|
|
368
|
+
return {
|
|
369
|
+
missingCount: tailFirstSeq - 1 - lastLogSeq,
|
|
370
|
+
tailFirstSeq,
|
|
371
|
+
};
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
/**
|
|
375
|
+
* Resolve which log lines this stream still has to emit, cursoring on
|
|
376
|
+
* absolute per-run sequence numbers. The snapshot only retains a rotated
|
|
377
|
+
* tail, so when the cursor has fallen behind the retained window the gap is
|
|
378
|
+
* surfaced as one loud marker line followed by the whole retained tail.
|
|
379
|
+
* Transports that can read the durable Run Log Stream resolve the gap with
|
|
380
|
+
* `resolvePlayRunLogGap` + a log-page read BEFORE diffing, so this marker
|
|
381
|
+
* path remains only for first-connects to in-flight runs and failed
|
|
382
|
+
* page reads (ADR-0009).
|
|
383
|
+
*
|
|
384
|
+
* `firstSeq` is set only when `lines` is a contiguous seq run (no marker).
|
|
385
|
+
*/
|
|
386
|
+
function diffLogLines(input: {
|
|
387
|
+
snapshot: PlayRunLiveSnapshot;
|
|
388
|
+
lastLogSeq: number;
|
|
389
|
+
}): { lines: string[]; lastLogSeq: number; firstSeq: number | null } {
|
|
390
|
+
const { logs, totalLogCount } = input.snapshot;
|
|
391
|
+
if (totalLogCount <= input.lastLogSeq) {
|
|
392
|
+
return { lines: [], lastLogSeq: input.lastLogSeq, firstSeq: null };
|
|
393
|
+
}
|
|
394
|
+
// Absolute sequence (1-based) of the first retained tail line.
|
|
395
|
+
const tailFirstSeq = totalLogCount - logs.length + 1;
|
|
396
|
+
if (input.lastLogSeq + 1 >= tailFirstSeq) {
|
|
397
|
+
return {
|
|
398
|
+
lines: logs.slice(input.lastLogSeq + 1 - tailFirstSeq),
|
|
399
|
+
lastLogSeq: totalLogCount,
|
|
400
|
+
firstSeq: input.lastLogSeq + 1,
|
|
401
|
+
};
|
|
402
|
+
}
|
|
403
|
+
const missingCount = tailFirstSeq - 1 - input.lastLogSeq;
|
|
404
|
+
return {
|
|
405
|
+
lines: [
|
|
406
|
+
`[stream] ${missingCount} log lines not retained in the live window; full logs via runs logs`,
|
|
407
|
+
...logs,
|
|
408
|
+
],
|
|
409
|
+
lastLogSeq: totalLogCount,
|
|
410
|
+
firstSeq: null,
|
|
411
|
+
};
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
export function diffPlayRunStreamEvents(input: {
|
|
415
|
+
streamId: string;
|
|
416
|
+
snapshot: PlayRunLiveSnapshot;
|
|
417
|
+
previous: PlayRunStreamDiffState;
|
|
418
|
+
}): {
|
|
419
|
+
events: PlayRunStreamEvent[];
|
|
420
|
+
next: PlayRunStreamDiffState;
|
|
421
|
+
} {
|
|
422
|
+
const { snapshot, streamId, previous } = input;
|
|
423
|
+
const cursor = getSnapshotCursor(snapshot);
|
|
424
|
+
const logDiff = diffLogLines({
|
|
425
|
+
snapshot,
|
|
426
|
+
lastLogSeq: previous.lastLogSeq,
|
|
427
|
+
});
|
|
428
|
+
const next: PlayRunStreamDiffState = {
|
|
429
|
+
runSignature: getRunSignature(snapshot),
|
|
430
|
+
stepStatusSignature: getStepStatusSignature(snapshot),
|
|
431
|
+
stepProgressSignature: getStepProgressSignature(snapshot),
|
|
432
|
+
snapshotSignature: getSnapshotSignature(snapshot),
|
|
433
|
+
lastLogSeq: logDiff.lastLogSeq,
|
|
434
|
+
};
|
|
435
|
+
const events: PlayRunStreamEvent[] = [];
|
|
436
|
+
|
|
437
|
+
if (next.stepStatusSignature !== previous.stepStatusSignature) {
|
|
438
|
+
for (const state of snapshot.nodeStates) {
|
|
439
|
+
if (state.status === 'idle') {
|
|
440
|
+
continue;
|
|
441
|
+
}
|
|
442
|
+
const persistedStartedAt =
|
|
443
|
+
state.startedAt ?? state.progress?.startedAt ?? null;
|
|
444
|
+
const persistedCompletedAt =
|
|
445
|
+
state.completedAt ?? state.progress?.completedAt ?? null;
|
|
446
|
+
|
|
447
|
+
events.push(
|
|
448
|
+
makeRunStreamEvent({
|
|
449
|
+
cursor,
|
|
450
|
+
streamId,
|
|
451
|
+
type: 'play.step.status',
|
|
452
|
+
payload: {
|
|
453
|
+
runId: snapshot.runId,
|
|
454
|
+
stepId: state.nodeId,
|
|
455
|
+
status: state.status,
|
|
456
|
+
artifactTableNamespace: state.artifactTableNamespace ?? null,
|
|
457
|
+
...resolveTimingWindow({
|
|
458
|
+
startedAt: persistedStartedAt,
|
|
459
|
+
completedAt: persistedCompletedAt,
|
|
460
|
+
updatedAt:
|
|
461
|
+
state.updatedAt ??
|
|
462
|
+
state.progress?.updatedAt ??
|
|
463
|
+
snapshot.updatedAt ??
|
|
464
|
+
null,
|
|
465
|
+
}),
|
|
466
|
+
},
|
|
467
|
+
}),
|
|
468
|
+
);
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
if (next.stepProgressSignature !== previous.stepProgressSignature) {
|
|
473
|
+
for (const state of snapshot.nodeStates) {
|
|
474
|
+
if (!state.progress) {
|
|
475
|
+
continue;
|
|
476
|
+
}
|
|
477
|
+
events.push(
|
|
478
|
+
makeRunStreamEvent({
|
|
479
|
+
cursor: String(
|
|
480
|
+
state.progress.updatedAt ?? snapshot.updatedAt ?? Date.now(),
|
|
481
|
+
),
|
|
482
|
+
streamId,
|
|
483
|
+
type: 'play.step.progress',
|
|
484
|
+
payload: {
|
|
485
|
+
runId: snapshot.runId,
|
|
486
|
+
stepId: state.nodeId,
|
|
487
|
+
completed: state.progress.completed,
|
|
488
|
+
total: state.progress.total,
|
|
489
|
+
failed: state.progress.failed,
|
|
490
|
+
message: state.progress.message,
|
|
491
|
+
artifactTableNamespace:
|
|
492
|
+
state.progress.artifactTableNamespace ??
|
|
493
|
+
state.artifactTableNamespace ??
|
|
494
|
+
null,
|
|
495
|
+
...resolveTimingWindow({
|
|
496
|
+
startedAt: state.startedAt ?? state.progress.startedAt ?? null,
|
|
497
|
+
completedAt:
|
|
498
|
+
state.completedAt ?? state.progress.completedAt ?? null,
|
|
499
|
+
updatedAt: state.progress.updatedAt ?? snapshot.updatedAt ?? null,
|
|
500
|
+
}),
|
|
501
|
+
},
|
|
502
|
+
}),
|
|
503
|
+
);
|
|
504
|
+
}
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
if (logDiff.lines.length > 0) {
|
|
508
|
+
events.push(
|
|
509
|
+
makeRunStreamEvent({
|
|
510
|
+
cursor,
|
|
511
|
+
streamId,
|
|
512
|
+
type: 'play.run.log',
|
|
513
|
+
payload: {
|
|
514
|
+
runId: snapshot.runId,
|
|
515
|
+
lines: logDiff.lines,
|
|
516
|
+
source: 'worker',
|
|
517
|
+
...(logDiff.firstSeq !== null ? { firstSeq: logDiff.firstSeq } : {}),
|
|
518
|
+
totalLogCount: snapshot.totalLogCount,
|
|
519
|
+
},
|
|
520
|
+
}),
|
|
521
|
+
);
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
if (next.snapshotSignature !== previous.snapshotSignature) {
|
|
525
|
+
const enrichedNodeStates = snapshot.nodeStates.map((state) => {
|
|
526
|
+
const timing = resolveTimingWindow({
|
|
527
|
+
startedAt: state.startedAt ?? state.progress?.startedAt ?? null,
|
|
528
|
+
completedAt: state.completedAt ?? state.progress?.completedAt ?? null,
|
|
529
|
+
updatedAt:
|
|
530
|
+
state.updatedAt ??
|
|
531
|
+
state.progress?.updatedAt ??
|
|
532
|
+
snapshot.updatedAt ??
|
|
533
|
+
null,
|
|
534
|
+
});
|
|
535
|
+
return {
|
|
536
|
+
...state,
|
|
537
|
+
...timing,
|
|
538
|
+
progress: state.progress
|
|
539
|
+
? {
|
|
540
|
+
...state.progress,
|
|
541
|
+
...resolveTimingWindow({
|
|
542
|
+
startedAt: state.progress.startedAt ?? state.startedAt ?? null,
|
|
543
|
+
completedAt:
|
|
544
|
+
state.progress.completedAt ?? state.completedAt ?? null,
|
|
545
|
+
updatedAt:
|
|
546
|
+
state.progress.updatedAt ??
|
|
547
|
+
state.updatedAt ??
|
|
548
|
+
snapshot.updatedAt ??
|
|
549
|
+
null,
|
|
550
|
+
}),
|
|
551
|
+
}
|
|
552
|
+
: state.progress,
|
|
553
|
+
};
|
|
554
|
+
});
|
|
555
|
+
events.push(
|
|
556
|
+
makeRunStreamEvent({
|
|
557
|
+
cursor,
|
|
558
|
+
streamId,
|
|
559
|
+
type: 'play.run.snapshot',
|
|
560
|
+
payload: { ...snapshot, nodeStates: enrichedNodeStates },
|
|
561
|
+
}),
|
|
562
|
+
);
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
if (next.runSignature !== previous.runSignature) {
|
|
566
|
+
events.push(
|
|
567
|
+
makeRunStreamEvent({
|
|
568
|
+
cursor,
|
|
569
|
+
streamId,
|
|
570
|
+
type: 'play.run.status',
|
|
571
|
+
payload: {
|
|
572
|
+
runId: snapshot.runId,
|
|
573
|
+
status: snapshot.status,
|
|
574
|
+
updatedAt: snapshot.updatedAt,
|
|
575
|
+
},
|
|
576
|
+
}),
|
|
577
|
+
);
|
|
578
|
+
}
|
|
579
|
+
|
|
580
|
+
return { events, next };
|
|
581
|
+
}
|
|
@@ -1029,6 +1029,50 @@ export function isToolExecuteResult(
|
|
|
1029
1029
|
);
|
|
1030
1030
|
}
|
|
1031
1031
|
|
|
1032
|
+
function resultRootOf(result: ToolExecuteResult): unknown {
|
|
1033
|
+
return { toolResponse: result.toolOutput };
|
|
1034
|
+
}
|
|
1035
|
+
|
|
1036
|
+
/**
|
|
1037
|
+
* Read a single value out of a tool result — by declared getter name, then by
|
|
1038
|
+
* selector paths, then by key. Used by SDK play helpers (`extractValue`).
|
|
1039
|
+
*/
|
|
1040
|
+
export function readValue(
|
|
1041
|
+
result: ToolExecuteResult,
|
|
1042
|
+
selector: readonly string[] | string,
|
|
1043
|
+
): unknown {
|
|
1044
|
+
if (typeof selector === 'string') {
|
|
1045
|
+
const declared = result.extractedValues[selector]?.get();
|
|
1046
|
+
if (declared != null) return declared;
|
|
1047
|
+
}
|
|
1048
|
+
const root = resultRootOf(result);
|
|
1049
|
+
const paths = Array.isArray(selector) ? selector : [selector];
|
|
1050
|
+
const byPath = findFirstTargetByPath(root, paths);
|
|
1051
|
+
if (byPath) return byPath.value;
|
|
1052
|
+
if (typeof selector === 'string') {
|
|
1053
|
+
const byKey = findFirstTargetByKey(root, selector);
|
|
1054
|
+
if (byKey) return byKey.value;
|
|
1055
|
+
}
|
|
1056
|
+
return null;
|
|
1057
|
+
}
|
|
1058
|
+
|
|
1059
|
+
/**
|
|
1060
|
+
* Read array rows out of a tool result — by selector paths, else the first
|
|
1061
|
+
* declared list accessor. Companion to {@link readValue} for `extractList`.
|
|
1062
|
+
*/
|
|
1063
|
+
export function readList(
|
|
1064
|
+
result: ToolExecuteResult,
|
|
1065
|
+
selector?: readonly string[] | string,
|
|
1066
|
+
): Record<string, unknown>[] {
|
|
1067
|
+
if (selector) {
|
|
1068
|
+
const paths = Array.isArray(selector) ? selector : [selector];
|
|
1069
|
+
const found = findFirstTargetByPath(resultRootOf(result), paths)?.value;
|
|
1070
|
+
const rows = normalizeRows(found);
|
|
1071
|
+
if (rows) return rows;
|
|
1072
|
+
}
|
|
1073
|
+
return Object.values(result.extractedLists)[0]?.get() ?? [];
|
|
1074
|
+
}
|
|
1075
|
+
|
|
1032
1076
|
function metadataInputFromToolExecuteResult(
|
|
1033
1077
|
value: ToolExecuteResult,
|
|
1034
1078
|
): ToolResultMetadataInput {
|
|
@@ -5,8 +5,7 @@ const PRIVATE_KEY_PATTERN =
|
|
|
5
5
|
const BEARER_LITERAL_PATTERN = /\bBearer\s+[A-Za-z0-9._~+/=-]{16,}/i;
|
|
6
6
|
const ASSIGNMENT_SECRET_LITERAL_PATTERN =
|
|
7
7
|
/\b(?:api[_-]?key|token|secret|password)\b\s*[:=]\s*['"][^'"]{12,}['"]/i;
|
|
8
|
-
const HIGH_ENTROPY_LITERAL_PATTERN =
|
|
9
|
-
/['"]([A-Za-z0-9+/=_-]{32,})['"]/g;
|
|
8
|
+
const HIGH_ENTROPY_LITERAL_PATTERN = /['"]([A-Za-z0-9+/=_-]{32,})['"]/g;
|
|
10
9
|
|
|
11
10
|
function shannonEntropy(value: string): number {
|
|
12
11
|
const counts = new Map<string, number>();
|
|
@@ -17,26 +16,38 @@ function shannonEntropy(value: string): number {
|
|
|
17
16
|
}, 0);
|
|
18
17
|
}
|
|
19
18
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
19
|
+
/**
|
|
20
|
+
* Returns the inline-secret findings in a string (empty if none). The throwing
|
|
21
|
+
* validator below and the workflows→plays migration validator both call this so
|
|
22
|
+
* the heuristics stay a single source of truth (no drift between "publish
|
|
23
|
+
* rejects it" and "transform skips it loudly").
|
|
24
|
+
*/
|
|
25
|
+
export function collectInlineSecretFindings(sourceCode: string): string[] {
|
|
24
26
|
const findings: string[] = [];
|
|
25
|
-
for (const match of
|
|
27
|
+
for (const match of sourceCode.matchAll(SECRET_ENV_PATTERN)) {
|
|
26
28
|
findings.push(`process.env.${match[1]}`);
|
|
27
29
|
}
|
|
28
|
-
if (PRIVATE_KEY_PATTERN.test(
|
|
29
|
-
if (BEARER_LITERAL_PATTERN.test(
|
|
30
|
-
|
|
30
|
+
if (PRIVATE_KEY_PATTERN.test(sourceCode)) findings.push('private key block');
|
|
31
|
+
if (BEARER_LITERAL_PATTERN.test(sourceCode))
|
|
32
|
+
findings.push('bearer token literal');
|
|
33
|
+
if (ASSIGNMENT_SECRET_LITERAL_PATTERN.test(sourceCode)) {
|
|
31
34
|
findings.push('secret-looking assignment literal');
|
|
32
35
|
}
|
|
33
|
-
for (const match of
|
|
36
|
+
for (const match of sourceCode.matchAll(HIGH_ENTROPY_LITERAL_PATTERN)) {
|
|
34
37
|
const literal = match[1] ?? '';
|
|
35
38
|
if (literal.length >= 40 && shannonEntropy(literal) >= 4.2) {
|
|
36
39
|
findings.push('high-entropy string literal');
|
|
37
40
|
break;
|
|
38
41
|
}
|
|
39
42
|
}
|
|
43
|
+
return [...new Set(findings)];
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export function validatePlaySourceHasNoInlineSecrets(input: {
|
|
47
|
+
sourceCode: string;
|
|
48
|
+
filePath: string;
|
|
49
|
+
}): void {
|
|
50
|
+
const findings = collectInlineSecretFindings(input.sourceCode);
|
|
40
51
|
if (!findings.length) return;
|
|
41
52
|
throw new Error(
|
|
42
53
|
[
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "deepline",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.94",
|
|
4
4
|
"description": "Deepline SDK + CLI — B2B data enrichment powered by durable cloud execution",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -50,11 +50,14 @@
|
|
|
50
50
|
"acorn": "^8.16.0",
|
|
51
51
|
"acorn-walk": "^8.3.5",
|
|
52
52
|
"commander": "^14.0.3",
|
|
53
|
+
"convex": "^1.32.0",
|
|
53
54
|
"csv-parse": "^5.6.0",
|
|
54
55
|
"csv-stringify": "^6.5.0",
|
|
55
|
-
"esbuild": "^0.25.11"
|
|
56
|
+
"esbuild": "^0.25.11",
|
|
57
|
+
"ws": "^8.18.0"
|
|
56
58
|
},
|
|
57
59
|
"devDependencies": {
|
|
60
|
+
"@types/ws": "^8.18.0",
|
|
58
61
|
"tsup": "^8.0.0",
|
|
59
62
|
"typescript": "^5.9.3"
|
|
60
63
|
}
|