deepline 0.1.12 → 0.1.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/README.md +14 -6
  2. package/dist/cli/index.js +1346 -717
  3. package/dist/cli/index.mjs +1342 -713
  4. package/dist/index.d.mts +199 -23
  5. package/dist/index.d.ts +199 -23
  6. package/dist/index.js +221 -14
  7. package/dist/index.mjs +221 -14
  8. package/dist/repo/apps/play-runner-workers/src/coordinator-entry.ts +214 -77
  9. package/dist/repo/apps/play-runner-workers/src/dedup-do.ts +85 -60
  10. package/dist/repo/apps/play-runner-workers/src/entry.ts +385 -66
  11. package/dist/repo/sdk/src/client.ts +237 -0
  12. package/dist/repo/sdk/src/config.ts +125 -8
  13. package/dist/repo/sdk/src/http.ts +29 -5
  14. package/dist/repo/sdk/src/play.ts +19 -36
  15. package/dist/repo/sdk/src/plays/bundle-play-file.ts +22 -8
  16. package/dist/repo/sdk/src/plays/local-file-discovery.ts +207 -160
  17. package/dist/repo/sdk/src/types.ts +25 -0
  18. package/dist/repo/sdk/src/version.ts +2 -2
  19. package/dist/repo/shared_libs/play-runtime/tool-result.ts +237 -145
  20. package/dist/repo/shared_libs/plays/bundling/index.ts +206 -229
  21. package/dist/repo/shared_libs/plays/dataset.ts +28 -0
  22. package/dist/repo/shared_libs/plays/row-identity.ts +59 -4
  23. package/package.json +5 -4
  24. package/dist/cli/index.js.map +0 -1
  25. package/dist/cli/index.mjs.map +0 -1
  26. package/dist/index.js.map +0 -1
  27. package/dist/index.mjs.map +0 -1
  28. package/dist/repo/apps/play-runner-workers/src/runtime/README.md +0 -21
  29. package/dist/repo/apps/play-runner-workers/src/runtime/batching.ts +0 -177
  30. package/dist/repo/apps/play-runner-workers/src/runtime/execution-plan.ts +0 -52
  31. package/dist/repo/apps/play-runner-workers/src/runtime/tool-batch.ts +0 -100
  32. package/dist/repo/sdk/src/cli/commands/auth.ts +0 -500
  33. package/dist/repo/sdk/src/cli/commands/billing.ts +0 -188
  34. package/dist/repo/sdk/src/cli/commands/csv.ts +0 -123
  35. package/dist/repo/sdk/src/cli/commands/db.ts +0 -119
  36. package/dist/repo/sdk/src/cli/commands/feedback.ts +0 -40
  37. package/dist/repo/sdk/src/cli/commands/org.ts +0 -117
  38. package/dist/repo/sdk/src/cli/commands/play.ts +0 -3441
  39. package/dist/repo/sdk/src/cli/commands/tools.ts +0 -687
  40. package/dist/repo/sdk/src/cli/dataset-stats.ts +0 -415
  41. package/dist/repo/sdk/src/cli/index.ts +0 -148
  42. package/dist/repo/sdk/src/cli/progress.ts +0 -149
  43. package/dist/repo/sdk/src/cli/skills-sync.ts +0 -141
  44. package/dist/repo/sdk/src/cli/trace.ts +0 -61
  45. package/dist/repo/sdk/src/cli/utils.ts +0 -145
  46. package/dist/repo/sdk/src/compat.ts +0 -77
  47. package/dist/repo/shared_libs/observability/node-tracing.ts +0 -129
  48. package/dist/repo/shared_libs/observability/tracing.ts +0 -98
  49. package/dist/repo/shared_libs/play-runtime/context.ts +0 -4242
  50. package/dist/repo/shared_libs/play-runtime/ctx-contract.ts +0 -250
  51. package/dist/repo/shared_libs/play-runtime/ctx-types.ts +0 -725
  52. package/dist/repo/shared_libs/play-runtime/dataset-id.ts +0 -10
  53. package/dist/repo/shared_libs/play-runtime/db-session-crypto.ts +0 -304
  54. package/dist/repo/shared_libs/play-runtime/db-session.ts +0 -462
  55. package/dist/repo/shared_libs/play-runtime/live-events.ts +0 -214
  56. package/dist/repo/shared_libs/play-runtime/live-state-contract.ts +0 -50
  57. package/dist/repo/shared_libs/play-runtime/map-execution-frame.ts +0 -114
  58. package/dist/repo/shared_libs/play-runtime/map-row-identity.ts +0 -158
  59. package/dist/repo/shared_libs/play-runtime/progress-emitter.ts +0 -172
  60. package/dist/repo/shared_libs/play-runtime/protocol.ts +0 -121
  61. package/dist/repo/shared_libs/play-runtime/public-play-contract.ts +0 -42
  62. package/dist/repo/shared_libs/play-runtime/result-normalization.ts +0 -33
  63. package/dist/repo/shared_libs/play-runtime/runtime-api.ts +0 -1873
  64. package/dist/repo/shared_libs/play-runtime/runtime-constraints.ts +0 -2
  65. package/dist/repo/shared_libs/play-runtime/runtime-pg-driver-neon-serverless.ts +0 -201
  66. package/dist/repo/shared_libs/play-runtime/runtime-pg-driver-pg.ts +0 -48
  67. package/dist/repo/shared_libs/play-runtime/runtime-pg-driver.ts +0 -84
  68. package/dist/repo/shared_libs/play-runtime/static-pipeline-types.ts +0 -147
  69. package/dist/repo/shared_libs/play-runtime/suspension.ts +0 -68
  70. package/dist/repo/shared_libs/play-runtime/tracing.ts +0 -31
  71. package/dist/repo/shared_libs/play-runtime/waterfall-replay.ts +0 -75
  72. package/dist/repo/shared_libs/play-runtime/worker-api-types.ts +0 -140
  73. package/dist/repo/shared_libs/plays/artifact-transport.ts +0 -14
  74. package/dist/repo/shared_libs/plays/artifact-types.ts +0 -49
  75. package/dist/repo/shared_libs/plays/compiler-manifest.ts +0 -186
  76. package/dist/repo/shared_libs/plays/definition.ts +0 -264
  77. package/dist/repo/shared_libs/plays/file-refs.ts +0 -11
  78. package/dist/repo/shared_libs/plays/rate-limit-scheduler.ts +0 -206
  79. package/dist/repo/shared_libs/plays/resolve-static-pipeline.ts +0 -164
  80. package/dist/repo/shared_libs/plays/runtime-validation.ts +0 -395
  81. package/dist/repo/shared_libs/temporal/constants.ts +0 -39
  82. package/dist/repo/shared_libs/temporal/preview-config.ts +0 -153
@@ -1,10 +0,0 @@
1
- import { createHash } from 'node:crypto';
2
-
3
- export function createRuntimeDatasetId(
4
- playName: string,
5
- tableNamespace: string,
6
- ): string {
7
- return createHash('sha1')
8
- .update(`${playName}:${tableNamespace}`)
9
- .digest('hex');
10
- }
@@ -1,304 +0,0 @@
1
- import type { CreateDbSessionResponse } from './db-session';
2
-
3
- const POSTGRES_URL_ENCRYPTION_ALGORITHM = 'AES-GCM' as const;
4
- const POSTGRES_URL_ENCRYPTION_KEY_ID =
5
- 'deepline-runtime-db-session-url:v1' as const;
6
- const POSTGRES_URL_PUBLIC_KEY_ENCRYPTION_KEY_ID =
7
- 'deepline-runtime-db-session-url:v2' as const;
8
- const POSTGRES_URL_ENCRYPTION_LABEL =
9
- 'deepline:runtime-db-session-postgres-url:v1';
10
- const POSTGRES_URL_PUBLIC_KEY_ENCRYPTION_ALGORITHM =
11
- 'RSA-OAEP-256+A256GCM' as const;
12
- const IV_LENGTH_BYTES = 12;
13
- const AUTH_TAG_LENGTH_BYTES = 16;
14
-
15
- export type SharedSecretEncryptedPostgresUrl = {
16
- alg: 'A256GCM';
17
- kid: typeof POSTGRES_URL_ENCRYPTION_KEY_ID;
18
- iv: string;
19
- ciphertext: string;
20
- tag: string;
21
- };
22
-
23
- export type PublicKeyEncryptedPostgresUrl = {
24
- alg: typeof POSTGRES_URL_PUBLIC_KEY_ENCRYPTION_ALGORITHM;
25
- kid: typeof POSTGRES_URL_PUBLIC_KEY_ENCRYPTION_KEY_ID;
26
- wrappedKey: string;
27
- iv: string;
28
- ciphertext: string;
29
- tag: string;
30
- };
31
-
32
- export type EncryptedPostgresUrl =
33
- | SharedSecretEncryptedPostgresUrl
34
- | PublicKeyEncryptedPostgresUrl;
35
-
36
- export type PostgresUrlEncryptionRequest = {
37
- alg: typeof POSTGRES_URL_PUBLIC_KEY_ENCRYPTION_ALGORITHM;
38
- publicKeyJwk: JsonWebKey;
39
- };
40
-
41
- export type PostgresUrlDecryptionKey = {
42
- request: PostgresUrlEncryptionRequest;
43
- privateKey: CryptoKey;
44
- };
45
-
46
- function encodeBase64Url(bytes: Uint8Array): string {
47
- const base64 =
48
- typeof Buffer !== 'undefined'
49
- ? Buffer.from(bytes).toString('base64')
50
- : btoa(String.fromCharCode(...bytes));
51
- return base64.replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/g, '');
52
- }
53
-
54
- function decodeBase64Url(value: string): Uint8Array {
55
- const padding =
56
- value.length % 4 === 0 ? '' : '='.repeat(4 - (value.length % 4));
57
- const normalized = value.replace(/-/g, '+').replace(/_/g, '/') + padding;
58
- if (typeof Buffer !== 'undefined') {
59
- return new Uint8Array(Buffer.from(normalized, 'base64'));
60
- }
61
- return Uint8Array.from(atob(normalized), (char) => char.charCodeAt(0));
62
- }
63
-
64
- function toArrayBuffer(bytes: Uint8Array): ArrayBuffer {
65
- return bytes.buffer.slice(
66
- bytes.byteOffset,
67
- bytes.byteOffset + bytes.byteLength,
68
- ) as ArrayBuffer;
69
- }
70
-
71
- type RsaOaepKeyGenAlgorithm = {
72
- name: 'RSA-OAEP';
73
- modulusLength: number;
74
- publicExponent: Uint8Array;
75
- hash: string;
76
- };
77
-
78
- type RsaOaepImportAlgorithm = {
79
- name: 'RSA-OAEP';
80
- hash: string;
81
- };
82
-
83
- function rsaOaepAlgorithm(): RsaOaepKeyGenAlgorithm {
84
- return {
85
- name: 'RSA-OAEP',
86
- modulusLength: 2048,
87
- publicExponent: new Uint8Array([1, 0, 1]),
88
- hash: 'SHA-256',
89
- };
90
- }
91
-
92
- function rsaOaepImportAlgorithm(): RsaOaepImportAlgorithm {
93
- return {
94
- name: 'RSA-OAEP',
95
- hash: 'SHA-256',
96
- };
97
- }
98
-
99
- async function derivePostgresUrlEncryptionKey(secret: string): Promise<CryptoKey> {
100
- const normalizedSecret = secret.trim();
101
- if (!normalizedSecret) {
102
- throw new Error('Runtime DB session encryption secret is empty.');
103
- }
104
- const keyBytes = new Uint8Array(
105
- await crypto.subtle.digest(
106
- 'SHA-256',
107
- new TextEncoder().encode(
108
- `${POSTGRES_URL_ENCRYPTION_LABEL}:${normalizedSecret}`,
109
- ),
110
- ),
111
- );
112
- return await crypto.subtle.importKey(
113
- 'raw',
114
- toArrayBuffer(keyBytes),
115
- { name: POSTGRES_URL_ENCRYPTION_ALGORITHM },
116
- false,
117
- ['encrypt', 'decrypt'],
118
- );
119
- }
120
-
121
- export function dbSessionPostgresUrlAad(
122
- session: Omit<CreateDbSessionResponse, 'postgresUrl' | 'encryptedPostgresUrl'>,
123
- ): string {
124
- return JSON.stringify({
125
- sessionId: session.sessionId,
126
- expiresAt: session.expiresAt,
127
- playName: session.playName,
128
- target: session.target,
129
- operations: [...session.operations].sort(),
130
- postgres: session.postgres ?? null,
131
- });
132
- }
133
-
134
- export async function generateDbSessionPostgresUrlDecryptionKey(): Promise<PostgresUrlDecryptionKey> {
135
- const keyPair = (await crypto.subtle.generateKey(
136
- rsaOaepAlgorithm(),
137
- true,
138
- ['encrypt', 'decrypt'],
139
- )) as CryptoKeyPair;
140
- const publicKeyJwk = (await crypto.subtle.exportKey(
141
- 'jwk',
142
- keyPair.publicKey,
143
- )) as JsonWebKey;
144
- return {
145
- request: {
146
- alg: POSTGRES_URL_PUBLIC_KEY_ENCRYPTION_ALGORITHM,
147
- publicKeyJwk,
148
- },
149
- privateKey: keyPair.privateKey,
150
- };
151
- }
152
-
153
- export async function encryptDbSessionPostgresUrl(input: {
154
- postgresUrl: string;
155
- secret: string;
156
- aad: string;
157
- }): Promise<SharedSecretEncryptedPostgresUrl> {
158
- const iv = crypto.getRandomValues(new Uint8Array(IV_LENGTH_BYTES));
159
- const key = await derivePostgresUrlEncryptionKey(input.secret);
160
- const encrypted = new Uint8Array(
161
- await crypto.subtle.encrypt(
162
- {
163
- name: POSTGRES_URL_ENCRYPTION_ALGORITHM,
164
- iv: toArrayBuffer(iv),
165
- additionalData: new TextEncoder().encode(input.aad),
166
- tagLength: AUTH_TAG_LENGTH_BYTES * 8,
167
- },
168
- key,
169
- new TextEncoder().encode(input.postgresUrl),
170
- ),
171
- );
172
- return {
173
- alg: 'A256GCM',
174
- kid: POSTGRES_URL_ENCRYPTION_KEY_ID,
175
- iv: encodeBase64Url(iv),
176
- ciphertext: encodeBase64Url(encrypted.slice(0, -AUTH_TAG_LENGTH_BYTES)),
177
- tag: encodeBase64Url(encrypted.slice(-AUTH_TAG_LENGTH_BYTES)),
178
- };
179
- }
180
-
181
- export async function encryptDbSessionPostgresUrlWithPublicKey(input: {
182
- postgresUrl: string;
183
- request: PostgresUrlEncryptionRequest;
184
- aad: string;
185
- }): Promise<PublicKeyEncryptedPostgresUrl> {
186
- if (input.request.alg !== POSTGRES_URL_PUBLIC_KEY_ENCRYPTION_ALGORITHM) {
187
- throw new Error('Unsupported runtime DB session public-key envelope.');
188
- }
189
- const publicKey = (await crypto.subtle.importKey(
190
- 'jwk',
191
- input.request.publicKeyJwk,
192
- rsaOaepImportAlgorithm(),
193
- false,
194
- ['encrypt'],
195
- )) as CryptoKey;
196
- const contentKey = (await crypto.subtle.generateKey(
197
- { name: POSTGRES_URL_ENCRYPTION_ALGORITHM, length: 256 },
198
- true,
199
- ['encrypt', 'decrypt'],
200
- )) as CryptoKey;
201
- const iv = crypto.getRandomValues(new Uint8Array(IV_LENGTH_BYTES));
202
- const encrypted = new Uint8Array(
203
- await crypto.subtle.encrypt(
204
- {
205
- name: POSTGRES_URL_ENCRYPTION_ALGORITHM,
206
- iv: toArrayBuffer(iv),
207
- additionalData: new TextEncoder().encode(input.aad),
208
- tagLength: AUTH_TAG_LENGTH_BYTES * 8,
209
- },
210
- contentKey,
211
- new TextEncoder().encode(input.postgresUrl),
212
- ),
213
- );
214
- const rawContentKey = (await crypto.subtle.exportKey(
215
- 'raw',
216
- contentKey,
217
- )) as ArrayBuffer;
218
- const wrappedKey = new Uint8Array(
219
- await crypto.subtle.encrypt(
220
- { name: 'RSA-OAEP' },
221
- publicKey,
222
- rawContentKey,
223
- ),
224
- );
225
- return {
226
- alg: POSTGRES_URL_PUBLIC_KEY_ENCRYPTION_ALGORITHM,
227
- kid: POSTGRES_URL_PUBLIC_KEY_ENCRYPTION_KEY_ID,
228
- wrappedKey: encodeBase64Url(wrappedKey),
229
- iv: encodeBase64Url(iv),
230
- ciphertext: encodeBase64Url(encrypted.slice(0, -AUTH_TAG_LENGTH_BYTES)),
231
- tag: encodeBase64Url(encrypted.slice(-AUTH_TAG_LENGTH_BYTES)),
232
- };
233
- }
234
-
235
- export async function decryptDbSessionPostgresUrl(input: {
236
- encrypted: EncryptedPostgresUrl;
237
- secret: string;
238
- aad: string;
239
- }): Promise<string> {
240
- if (
241
- input.encrypted.alg !== 'A256GCM' ||
242
- input.encrypted.kid !== POSTGRES_URL_ENCRYPTION_KEY_ID
243
- ) {
244
- throw new Error('Unsupported runtime DB session URL encryption envelope.');
245
- }
246
- const ciphertext = decodeBase64Url(input.encrypted.ciphertext);
247
- const tag = decodeBase64Url(input.encrypted.tag);
248
- const combined = new Uint8Array(ciphertext.byteLength + tag.byteLength);
249
- combined.set(ciphertext, 0);
250
- combined.set(tag, ciphertext.byteLength);
251
- const key = await derivePostgresUrlEncryptionKey(input.secret);
252
- const plaintext = await crypto.subtle.decrypt(
253
- {
254
- name: POSTGRES_URL_ENCRYPTION_ALGORITHM,
255
- iv: toArrayBuffer(decodeBase64Url(input.encrypted.iv)),
256
- additionalData: new TextEncoder().encode(input.aad),
257
- tagLength: AUTH_TAG_LENGTH_BYTES * 8,
258
- },
259
- key,
260
- toArrayBuffer(combined),
261
- );
262
- return new TextDecoder().decode(plaintext);
263
- }
264
-
265
- export async function decryptDbSessionPostgresUrlWithPrivateKey(input: {
266
- encrypted: EncryptedPostgresUrl;
267
- privateKey: CryptoKey;
268
- aad: string;
269
- }): Promise<string> {
270
- if (
271
- input.encrypted.alg !== POSTGRES_URL_PUBLIC_KEY_ENCRYPTION_ALGORITHM ||
272
- input.encrypted.kid !== POSTGRES_URL_PUBLIC_KEY_ENCRYPTION_KEY_ID
273
- ) {
274
- throw new Error('Unsupported runtime DB session URL public-key envelope.');
275
- }
276
- const rawContentKey = await crypto.subtle.decrypt(
277
- { name: 'RSA-OAEP' },
278
- input.privateKey,
279
- toArrayBuffer(decodeBase64Url(input.encrypted.wrappedKey)),
280
- );
281
- const contentKey = await crypto.subtle.importKey(
282
- 'raw',
283
- rawContentKey,
284
- { name: POSTGRES_URL_ENCRYPTION_ALGORITHM },
285
- false,
286
- ['decrypt'],
287
- );
288
- const ciphertext = decodeBase64Url(input.encrypted.ciphertext);
289
- const tag = decodeBase64Url(input.encrypted.tag);
290
- const combined = new Uint8Array(ciphertext.byteLength + tag.byteLength);
291
- combined.set(ciphertext, 0);
292
- combined.set(tag, ciphertext.byteLength);
293
- const plaintext = await crypto.subtle.decrypt(
294
- {
295
- name: POSTGRES_URL_ENCRYPTION_ALGORITHM,
296
- iv: toArrayBuffer(decodeBase64Url(input.encrypted.iv)),
297
- additionalData: new TextEncoder().encode(input.aad),
298
- tagLength: AUTH_TAG_LENGTH_BYTES * 8,
299
- },
300
- contentKey,
301
- toArrayBuffer(combined),
302
- );
303
- return new TextDecoder().decode(plaintext);
304
- }