deepline 0.1.0 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/dist/cli/index.js +212 -54
  2. package/dist/cli/index.js.map +1 -1
  3. package/dist/cli/index.mjs +198 -40
  4. package/dist/cli/index.mjs.map +1 -1
  5. package/dist/index.d.mts +1 -1
  6. package/dist/index.d.ts +1 -1
  7. package/dist/index.js +1 -1
  8. package/dist/index.mjs +1 -1
  9. package/dist/repo/apps/play-runner-workers/src/coordinator-entry.ts +3256 -0
  10. package/dist/repo/apps/play-runner-workers/src/dedup-do.ts +710 -0
  11. package/dist/repo/apps/play-runner-workers/src/entry.ts +5070 -0
  12. package/dist/repo/apps/play-runner-workers/src/runtime/README.md +21 -0
  13. package/dist/repo/apps/play-runner-workers/src/runtime/batching.ts +177 -0
  14. package/dist/repo/apps/play-runner-workers/src/runtime/execution-plan.ts +52 -0
  15. package/dist/repo/apps/play-runner-workers/src/runtime/tool-batch.ts +100 -0
  16. package/dist/repo/apps/play-runner-workers/src/runtime/tool-result.ts +184 -0
  17. package/dist/repo/sdk/src/cli/commands/auth.ts +482 -0
  18. package/dist/repo/sdk/src/cli/commands/billing.ts +188 -0
  19. package/dist/repo/sdk/src/cli/commands/csv.ts +123 -0
  20. package/dist/repo/sdk/src/cli/commands/db.ts +119 -0
  21. package/dist/repo/sdk/src/cli/commands/feedback.ts +40 -0
  22. package/dist/repo/sdk/src/cli/commands/org.ts +117 -0
  23. package/dist/repo/sdk/src/cli/commands/play.ts +3200 -0
  24. package/dist/repo/sdk/src/cli/commands/tools.ts +687 -0
  25. package/dist/repo/sdk/src/cli/dataset-stats.ts +341 -0
  26. package/dist/repo/sdk/src/cli/index.ts +138 -0
  27. package/dist/repo/sdk/src/cli/progress.ts +135 -0
  28. package/dist/repo/sdk/src/cli/trace.ts +61 -0
  29. package/dist/repo/sdk/src/cli/utils.ts +145 -0
  30. package/dist/repo/sdk/src/client.ts +1188 -0
  31. package/dist/repo/sdk/src/compat.ts +77 -0
  32. package/dist/repo/sdk/src/config.ts +285 -0
  33. package/dist/repo/sdk/src/errors.ts +125 -0
  34. package/dist/repo/sdk/src/http.ts +391 -0
  35. package/dist/repo/sdk/src/index.ts +139 -0
  36. package/dist/repo/sdk/src/play.ts +1330 -0
  37. package/dist/repo/sdk/src/plays/bundle-play-file.ts +133 -0
  38. package/dist/repo/sdk/src/plays/harness-stub.ts +210 -0
  39. package/dist/repo/sdk/src/plays/local-file-discovery.ts +326 -0
  40. package/dist/repo/sdk/src/tool-output.ts +489 -0
  41. package/dist/repo/sdk/src/types.ts +669 -0
  42. package/dist/repo/sdk/src/version.ts +2 -0
  43. package/dist/repo/sdk/src/worker-play-entry.ts +286 -0
  44. package/dist/repo/shared_libs/observability/node-tracing.ts +129 -0
  45. package/dist/repo/shared_libs/observability/tracing.ts +98 -0
  46. package/dist/repo/shared_libs/play-runtime/backend.ts +139 -0
  47. package/dist/repo/shared_libs/play-runtime/batch-runtime.ts +182 -0
  48. package/dist/repo/shared_libs/play-runtime/batching-types.ts +91 -0
  49. package/dist/repo/shared_libs/play-runtime/context.ts +3999 -0
  50. package/dist/repo/shared_libs/play-runtime/coordinator-headers.ts +78 -0
  51. package/dist/repo/shared_libs/play-runtime/ctx-contract.ts +250 -0
  52. package/dist/repo/shared_libs/play-runtime/ctx-types.ts +713 -0
  53. package/dist/repo/shared_libs/play-runtime/dataset-id.ts +10 -0
  54. package/dist/repo/shared_libs/play-runtime/db-session-crypto.ts +304 -0
  55. package/dist/repo/shared_libs/play-runtime/db-session.ts +462 -0
  56. package/dist/repo/shared_libs/play-runtime/dedup-backend.ts +0 -0
  57. package/dist/repo/shared_libs/play-runtime/default-batch-strategies.ts +124 -0
  58. package/dist/repo/shared_libs/play-runtime/execution-plan.ts +262 -0
  59. package/dist/repo/shared_libs/play-runtime/live-events.ts +214 -0
  60. package/dist/repo/shared_libs/play-runtime/live-state-contract.ts +50 -0
  61. package/dist/repo/shared_libs/play-runtime/map-execution-frame.ts +114 -0
  62. package/dist/repo/shared_libs/play-runtime/map-row-identity.ts +158 -0
  63. package/dist/repo/shared_libs/play-runtime/profiles.ts +90 -0
  64. package/dist/repo/shared_libs/play-runtime/progress-emitter.ts +172 -0
  65. package/dist/repo/shared_libs/play-runtime/protocol.ts +121 -0
  66. package/dist/repo/shared_libs/play-runtime/public-play-contract.ts +42 -0
  67. package/dist/repo/shared_libs/play-runtime/result-normalization.ts +33 -0
  68. package/dist/repo/shared_libs/play-runtime/runtime-actions.ts +208 -0
  69. package/dist/repo/shared_libs/play-runtime/runtime-api.ts +1873 -0
  70. package/dist/repo/shared_libs/play-runtime/runtime-constraints.ts +2 -0
  71. package/dist/repo/shared_libs/play-runtime/runtime-pg-driver-neon-serverless.ts +201 -0
  72. package/dist/repo/shared_libs/play-runtime/runtime-pg-driver-pg.ts +48 -0
  73. package/dist/repo/shared_libs/play-runtime/runtime-pg-driver.ts +84 -0
  74. package/dist/repo/shared_libs/play-runtime/scheduler-backend.ts +174 -0
  75. package/dist/repo/shared_libs/play-runtime/static-pipeline-types.ts +147 -0
  76. package/dist/repo/shared_libs/play-runtime/suspension.ts +68 -0
  77. package/dist/repo/shared_libs/play-runtime/tool-batch-executor.ts +146 -0
  78. package/dist/repo/shared_libs/play-runtime/tool-result.ts +387 -0
  79. package/dist/repo/shared_libs/play-runtime/tracing.ts +31 -0
  80. package/dist/repo/shared_libs/play-runtime/waterfall-replay.ts +75 -0
  81. package/dist/repo/shared_libs/play-runtime/worker-api-types.ts +140 -0
  82. package/dist/repo/shared_libs/plays/artifact-transport.ts +14 -0
  83. package/dist/repo/shared_libs/plays/artifact-types.ts +49 -0
  84. package/dist/repo/shared_libs/plays/bundling/index.ts +1346 -0
  85. package/dist/repo/shared_libs/plays/compiler-manifest.ts +186 -0
  86. package/dist/repo/shared_libs/plays/contracts.ts +51 -0
  87. package/dist/repo/shared_libs/plays/dataset.ts +308 -0
  88. package/dist/repo/shared_libs/plays/definition.ts +264 -0
  89. package/dist/repo/shared_libs/plays/file-refs.ts +11 -0
  90. package/dist/repo/shared_libs/plays/rate-limit-scheduler.ts +206 -0
  91. package/dist/repo/shared_libs/plays/resolve-static-pipeline.ts +164 -0
  92. package/dist/repo/shared_libs/plays/row-identity.ts +302 -0
  93. package/dist/repo/shared_libs/plays/runtime-validation.ts +415 -0
  94. package/dist/repo/shared_libs/plays/static-pipeline.ts +560 -0
  95. package/dist/repo/shared_libs/temporal/constants.ts +39 -0
  96. package/dist/repo/shared_libs/temporal/preview-config.ts +153 -0
  97. package/package.json +4 -4
@@ -0,0 +1,304 @@
1
+ import type { CreateDbSessionResponse } from './db-session';
2
+
3
+ const POSTGRES_URL_ENCRYPTION_ALGORITHM = 'AES-GCM' as const;
4
+ const POSTGRES_URL_ENCRYPTION_KEY_ID =
5
+ 'deepline-runtime-db-session-url:v1' as const;
6
+ const POSTGRES_URL_PUBLIC_KEY_ENCRYPTION_KEY_ID =
7
+ 'deepline-runtime-db-session-url:v2' as const;
8
+ const POSTGRES_URL_ENCRYPTION_LABEL =
9
+ 'deepline:runtime-db-session-postgres-url:v1';
10
+ const POSTGRES_URL_PUBLIC_KEY_ENCRYPTION_ALGORITHM =
11
+ 'RSA-OAEP-256+A256GCM' as const;
12
+ const IV_LENGTH_BYTES = 12;
13
+ const AUTH_TAG_LENGTH_BYTES = 16;
14
+
15
+ export type SharedSecretEncryptedPostgresUrl = {
16
+ alg: 'A256GCM';
17
+ kid: typeof POSTGRES_URL_ENCRYPTION_KEY_ID;
18
+ iv: string;
19
+ ciphertext: string;
20
+ tag: string;
21
+ };
22
+
23
+ export type PublicKeyEncryptedPostgresUrl = {
24
+ alg: typeof POSTGRES_URL_PUBLIC_KEY_ENCRYPTION_ALGORITHM;
25
+ kid: typeof POSTGRES_URL_PUBLIC_KEY_ENCRYPTION_KEY_ID;
26
+ wrappedKey: string;
27
+ iv: string;
28
+ ciphertext: string;
29
+ tag: string;
30
+ };
31
+
32
+ export type EncryptedPostgresUrl =
33
+ | SharedSecretEncryptedPostgresUrl
34
+ | PublicKeyEncryptedPostgresUrl;
35
+
36
+ export type PostgresUrlEncryptionRequest = {
37
+ alg: typeof POSTGRES_URL_PUBLIC_KEY_ENCRYPTION_ALGORITHM;
38
+ publicKeyJwk: JsonWebKey;
39
+ };
40
+
41
+ export type PostgresUrlDecryptionKey = {
42
+ request: PostgresUrlEncryptionRequest;
43
+ privateKey: CryptoKey;
44
+ };
45
+
46
+ function encodeBase64Url(bytes: Uint8Array): string {
47
+ const base64 =
48
+ typeof Buffer !== 'undefined'
49
+ ? Buffer.from(bytes).toString('base64')
50
+ : btoa(String.fromCharCode(...bytes));
51
+ return base64.replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/g, '');
52
+ }
53
+
54
+ function decodeBase64Url(value: string): Uint8Array {
55
+ const padding =
56
+ value.length % 4 === 0 ? '' : '='.repeat(4 - (value.length % 4));
57
+ const normalized = value.replace(/-/g, '+').replace(/_/g, '/') + padding;
58
+ if (typeof Buffer !== 'undefined') {
59
+ return new Uint8Array(Buffer.from(normalized, 'base64'));
60
+ }
61
+ return Uint8Array.from(atob(normalized), (char) => char.charCodeAt(0));
62
+ }
63
+
64
+ function toArrayBuffer(bytes: Uint8Array): ArrayBuffer {
65
+ return bytes.buffer.slice(
66
+ bytes.byteOffset,
67
+ bytes.byteOffset + bytes.byteLength,
68
+ ) as ArrayBuffer;
69
+ }
70
+
71
+ type RsaOaepKeyGenAlgorithm = {
72
+ name: 'RSA-OAEP';
73
+ modulusLength: number;
74
+ publicExponent: Uint8Array;
75
+ hash: string;
76
+ };
77
+
78
+ type RsaOaepImportAlgorithm = {
79
+ name: 'RSA-OAEP';
80
+ hash: string;
81
+ };
82
+
83
+ function rsaOaepAlgorithm(): RsaOaepKeyGenAlgorithm {
84
+ return {
85
+ name: 'RSA-OAEP',
86
+ modulusLength: 2048,
87
+ publicExponent: new Uint8Array([1, 0, 1]),
88
+ hash: 'SHA-256',
89
+ };
90
+ }
91
+
92
+ function rsaOaepImportAlgorithm(): RsaOaepImportAlgorithm {
93
+ return {
94
+ name: 'RSA-OAEP',
95
+ hash: 'SHA-256',
96
+ };
97
+ }
98
+
99
+ async function derivePostgresUrlEncryptionKey(secret: string): Promise<CryptoKey> {
100
+ const normalizedSecret = secret.trim();
101
+ if (!normalizedSecret) {
102
+ throw new Error('Runtime DB session encryption secret is empty.');
103
+ }
104
+ const keyBytes = new Uint8Array(
105
+ await crypto.subtle.digest(
106
+ 'SHA-256',
107
+ new TextEncoder().encode(
108
+ `${POSTGRES_URL_ENCRYPTION_LABEL}:${normalizedSecret}`,
109
+ ),
110
+ ),
111
+ );
112
+ return await crypto.subtle.importKey(
113
+ 'raw',
114
+ toArrayBuffer(keyBytes),
115
+ { name: POSTGRES_URL_ENCRYPTION_ALGORITHM },
116
+ false,
117
+ ['encrypt', 'decrypt'],
118
+ );
119
+ }
120
+
121
+ export function dbSessionPostgresUrlAad(
122
+ session: Omit<CreateDbSessionResponse, 'postgresUrl' | 'encryptedPostgresUrl'>,
123
+ ): string {
124
+ return JSON.stringify({
125
+ sessionId: session.sessionId,
126
+ expiresAt: session.expiresAt,
127
+ playName: session.playName,
128
+ target: session.target,
129
+ operations: [...session.operations].sort(),
130
+ postgres: session.postgres ?? null,
131
+ });
132
+ }
133
+
134
+ export async function generateDbSessionPostgresUrlDecryptionKey(): Promise<PostgresUrlDecryptionKey> {
135
+ const keyPair = (await crypto.subtle.generateKey(
136
+ rsaOaepAlgorithm(),
137
+ true,
138
+ ['encrypt', 'decrypt'],
139
+ )) as CryptoKeyPair;
140
+ const publicKeyJwk = (await crypto.subtle.exportKey(
141
+ 'jwk',
142
+ keyPair.publicKey,
143
+ )) as JsonWebKey;
144
+ return {
145
+ request: {
146
+ alg: POSTGRES_URL_PUBLIC_KEY_ENCRYPTION_ALGORITHM,
147
+ publicKeyJwk,
148
+ },
149
+ privateKey: keyPair.privateKey,
150
+ };
151
+ }
152
+
153
+ export async function encryptDbSessionPostgresUrl(input: {
154
+ postgresUrl: string;
155
+ secret: string;
156
+ aad: string;
157
+ }): Promise<SharedSecretEncryptedPostgresUrl> {
158
+ const iv = crypto.getRandomValues(new Uint8Array(IV_LENGTH_BYTES));
159
+ const key = await derivePostgresUrlEncryptionKey(input.secret);
160
+ const encrypted = new Uint8Array(
161
+ await crypto.subtle.encrypt(
162
+ {
163
+ name: POSTGRES_URL_ENCRYPTION_ALGORITHM,
164
+ iv: toArrayBuffer(iv),
165
+ additionalData: new TextEncoder().encode(input.aad),
166
+ tagLength: AUTH_TAG_LENGTH_BYTES * 8,
167
+ },
168
+ key,
169
+ new TextEncoder().encode(input.postgresUrl),
170
+ ),
171
+ );
172
+ return {
173
+ alg: 'A256GCM',
174
+ kid: POSTGRES_URL_ENCRYPTION_KEY_ID,
175
+ iv: encodeBase64Url(iv),
176
+ ciphertext: encodeBase64Url(encrypted.slice(0, -AUTH_TAG_LENGTH_BYTES)),
177
+ tag: encodeBase64Url(encrypted.slice(-AUTH_TAG_LENGTH_BYTES)),
178
+ };
179
+ }
180
+
181
+ export async function encryptDbSessionPostgresUrlWithPublicKey(input: {
182
+ postgresUrl: string;
183
+ request: PostgresUrlEncryptionRequest;
184
+ aad: string;
185
+ }): Promise<PublicKeyEncryptedPostgresUrl> {
186
+ if (input.request.alg !== POSTGRES_URL_PUBLIC_KEY_ENCRYPTION_ALGORITHM) {
187
+ throw new Error('Unsupported runtime DB session public-key envelope.');
188
+ }
189
+ const publicKey = (await crypto.subtle.importKey(
190
+ 'jwk',
191
+ input.request.publicKeyJwk,
192
+ rsaOaepImportAlgorithm(),
193
+ false,
194
+ ['encrypt'],
195
+ )) as CryptoKey;
196
+ const contentKey = (await crypto.subtle.generateKey(
197
+ { name: POSTGRES_URL_ENCRYPTION_ALGORITHM, length: 256 },
198
+ true,
199
+ ['encrypt', 'decrypt'],
200
+ )) as CryptoKey;
201
+ const iv = crypto.getRandomValues(new Uint8Array(IV_LENGTH_BYTES));
202
+ const encrypted = new Uint8Array(
203
+ await crypto.subtle.encrypt(
204
+ {
205
+ name: POSTGRES_URL_ENCRYPTION_ALGORITHM,
206
+ iv: toArrayBuffer(iv),
207
+ additionalData: new TextEncoder().encode(input.aad),
208
+ tagLength: AUTH_TAG_LENGTH_BYTES * 8,
209
+ },
210
+ contentKey,
211
+ new TextEncoder().encode(input.postgresUrl),
212
+ ),
213
+ );
214
+ const rawContentKey = (await crypto.subtle.exportKey(
215
+ 'raw',
216
+ contentKey,
217
+ )) as ArrayBuffer;
218
+ const wrappedKey = new Uint8Array(
219
+ await crypto.subtle.encrypt(
220
+ { name: 'RSA-OAEP' },
221
+ publicKey,
222
+ rawContentKey,
223
+ ),
224
+ );
225
+ return {
226
+ alg: POSTGRES_URL_PUBLIC_KEY_ENCRYPTION_ALGORITHM,
227
+ kid: POSTGRES_URL_PUBLIC_KEY_ENCRYPTION_KEY_ID,
228
+ wrappedKey: encodeBase64Url(wrappedKey),
229
+ iv: encodeBase64Url(iv),
230
+ ciphertext: encodeBase64Url(encrypted.slice(0, -AUTH_TAG_LENGTH_BYTES)),
231
+ tag: encodeBase64Url(encrypted.slice(-AUTH_TAG_LENGTH_BYTES)),
232
+ };
233
+ }
234
+
235
+ export async function decryptDbSessionPostgresUrl(input: {
236
+ encrypted: EncryptedPostgresUrl;
237
+ secret: string;
238
+ aad: string;
239
+ }): Promise<string> {
240
+ if (
241
+ input.encrypted.alg !== 'A256GCM' ||
242
+ input.encrypted.kid !== POSTGRES_URL_ENCRYPTION_KEY_ID
243
+ ) {
244
+ throw new Error('Unsupported runtime DB session URL encryption envelope.');
245
+ }
246
+ const ciphertext = decodeBase64Url(input.encrypted.ciphertext);
247
+ const tag = decodeBase64Url(input.encrypted.tag);
248
+ const combined = new Uint8Array(ciphertext.byteLength + tag.byteLength);
249
+ combined.set(ciphertext, 0);
250
+ combined.set(tag, ciphertext.byteLength);
251
+ const key = await derivePostgresUrlEncryptionKey(input.secret);
252
+ const plaintext = await crypto.subtle.decrypt(
253
+ {
254
+ name: POSTGRES_URL_ENCRYPTION_ALGORITHM,
255
+ iv: toArrayBuffer(decodeBase64Url(input.encrypted.iv)),
256
+ additionalData: new TextEncoder().encode(input.aad),
257
+ tagLength: AUTH_TAG_LENGTH_BYTES * 8,
258
+ },
259
+ key,
260
+ toArrayBuffer(combined),
261
+ );
262
+ return new TextDecoder().decode(plaintext);
263
+ }
264
+
265
+ export async function decryptDbSessionPostgresUrlWithPrivateKey(input: {
266
+ encrypted: EncryptedPostgresUrl;
267
+ privateKey: CryptoKey;
268
+ aad: string;
269
+ }): Promise<string> {
270
+ if (
271
+ input.encrypted.alg !== POSTGRES_URL_PUBLIC_KEY_ENCRYPTION_ALGORITHM ||
272
+ input.encrypted.kid !== POSTGRES_URL_PUBLIC_KEY_ENCRYPTION_KEY_ID
273
+ ) {
274
+ throw new Error('Unsupported runtime DB session URL public-key envelope.');
275
+ }
276
+ const rawContentKey = await crypto.subtle.decrypt(
277
+ { name: 'RSA-OAEP' },
278
+ input.privateKey,
279
+ toArrayBuffer(decodeBase64Url(input.encrypted.wrappedKey)),
280
+ );
281
+ const contentKey = await crypto.subtle.importKey(
282
+ 'raw',
283
+ rawContentKey,
284
+ { name: POSTGRES_URL_ENCRYPTION_ALGORITHM },
285
+ false,
286
+ ['decrypt'],
287
+ );
288
+ const ciphertext = decodeBase64Url(input.encrypted.ciphertext);
289
+ const tag = decodeBase64Url(input.encrypted.tag);
290
+ const combined = new Uint8Array(ciphertext.byteLength + tag.byteLength);
291
+ combined.set(ciphertext, 0);
292
+ combined.set(tag, ciphertext.byteLength);
293
+ const plaintext = await crypto.subtle.decrypt(
294
+ {
295
+ name: POSTGRES_URL_ENCRYPTION_ALGORITHM,
296
+ iv: toArrayBuffer(decodeBase64Url(input.encrypted.iv)),
297
+ additionalData: new TextEncoder().encode(input.aad),
298
+ tagLength: AUTH_TAG_LENGTH_BYTES * 8,
299
+ },
300
+ contentKey,
301
+ toArrayBuffer(combined),
302
+ );
303
+ return new TextDecoder().decode(plaintext);
304
+ }