deepline 0.0.1 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +324 -0
- package/dist/cli/index.js +6750 -503
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/index.mjs +6735 -512
- package/dist/cli/index.mjs.map +1 -1
- package/dist/index.d.mts +2349 -32
- package/dist/index.d.ts +2349 -32
- package/dist/index.js +1631 -82
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1617 -83
- package/dist/index.mjs.map +1 -1
- package/dist/repo/apps/play-runner-workers/src/coordinator-entry.ts +3256 -0
- package/dist/repo/apps/play-runner-workers/src/dedup-do.ts +710 -0
- package/dist/repo/apps/play-runner-workers/src/entry.ts +5070 -0
- package/dist/repo/apps/play-runner-workers/src/runtime/README.md +21 -0
- package/dist/repo/apps/play-runner-workers/src/runtime/batching.ts +177 -0
- package/dist/repo/apps/play-runner-workers/src/runtime/execution-plan.ts +52 -0
- package/dist/repo/apps/play-runner-workers/src/runtime/tool-batch.ts +100 -0
- package/dist/repo/apps/play-runner-workers/src/runtime/tool-result.ts +184 -0
- package/dist/repo/sdk/src/cli/commands/auth.ts +482 -0
- package/dist/repo/sdk/src/cli/commands/billing.ts +188 -0
- package/dist/repo/sdk/src/cli/commands/csv.ts +123 -0
- package/dist/repo/sdk/src/cli/commands/db.ts +119 -0
- package/dist/repo/sdk/src/cli/commands/feedback.ts +40 -0
- package/dist/repo/sdk/src/cli/commands/org.ts +117 -0
- package/dist/repo/sdk/src/cli/commands/play.ts +3200 -0
- package/dist/repo/sdk/src/cli/commands/tools.ts +687 -0
- package/dist/repo/sdk/src/cli/dataset-stats.ts +341 -0
- package/dist/repo/sdk/src/cli/index.ts +138 -0
- package/dist/repo/sdk/src/cli/progress.ts +135 -0
- package/dist/repo/sdk/src/cli/trace.ts +61 -0
- package/dist/repo/sdk/src/cli/utils.ts +145 -0
- package/dist/repo/sdk/src/client.ts +1188 -0
- package/dist/repo/sdk/src/compat.ts +77 -0
- package/dist/repo/sdk/src/config.ts +285 -0
- package/dist/repo/sdk/src/errors.ts +125 -0
- package/dist/repo/sdk/src/http.ts +391 -0
- package/dist/repo/sdk/src/index.ts +139 -0
- package/dist/repo/sdk/src/play.ts +1330 -0
- package/dist/repo/sdk/src/plays/bundle-play-file.ts +133 -0
- package/dist/repo/sdk/src/plays/harness-stub.ts +210 -0
- package/dist/repo/sdk/src/plays/local-file-discovery.ts +326 -0
- package/dist/repo/sdk/src/tool-output.ts +489 -0
- package/dist/repo/sdk/src/types.ts +669 -0
- package/dist/repo/sdk/src/version.ts +2 -0
- package/dist/repo/sdk/src/worker-play-entry.ts +286 -0
- package/dist/repo/shared_libs/observability/node-tracing.ts +129 -0
- package/dist/repo/shared_libs/observability/tracing.ts +98 -0
- package/dist/repo/shared_libs/play-runtime/backend.ts +139 -0
- package/dist/repo/shared_libs/play-runtime/batch-runtime.ts +182 -0
- package/dist/repo/shared_libs/play-runtime/batching-types.ts +91 -0
- package/dist/repo/shared_libs/play-runtime/context.ts +3999 -0
- package/dist/repo/shared_libs/play-runtime/coordinator-headers.ts +78 -0
- package/dist/repo/shared_libs/play-runtime/ctx-contract.ts +250 -0
- package/dist/repo/shared_libs/play-runtime/ctx-types.ts +713 -0
- package/dist/repo/shared_libs/play-runtime/dataset-id.ts +10 -0
- package/dist/repo/shared_libs/play-runtime/db-session-crypto.ts +304 -0
- package/dist/repo/shared_libs/play-runtime/db-session.ts +462 -0
- package/dist/repo/shared_libs/play-runtime/dedup-backend.ts +0 -0
- package/dist/repo/shared_libs/play-runtime/default-batch-strategies.ts +124 -0
- package/dist/repo/shared_libs/play-runtime/execution-plan.ts +262 -0
- package/dist/repo/shared_libs/play-runtime/live-events.ts +214 -0
- package/dist/repo/shared_libs/play-runtime/live-state-contract.ts +50 -0
- package/dist/repo/shared_libs/play-runtime/map-execution-frame.ts +114 -0
- package/dist/repo/shared_libs/play-runtime/map-row-identity.ts +158 -0
- package/dist/repo/shared_libs/play-runtime/profiles.ts +90 -0
- package/dist/repo/shared_libs/play-runtime/progress-emitter.ts +172 -0
- package/dist/repo/shared_libs/play-runtime/protocol.ts +121 -0
- package/dist/repo/shared_libs/play-runtime/public-play-contract.ts +42 -0
- package/dist/repo/shared_libs/play-runtime/result-normalization.ts +33 -0
- package/dist/repo/shared_libs/play-runtime/runtime-actions.ts +208 -0
- package/dist/repo/shared_libs/play-runtime/runtime-api.ts +1873 -0
- package/dist/repo/shared_libs/play-runtime/runtime-constraints.ts +2 -0
- package/dist/repo/shared_libs/play-runtime/runtime-pg-driver-neon-serverless.ts +201 -0
- package/dist/repo/shared_libs/play-runtime/runtime-pg-driver-pg.ts +48 -0
- package/dist/repo/shared_libs/play-runtime/runtime-pg-driver.ts +84 -0
- package/dist/repo/shared_libs/play-runtime/scheduler-backend.ts +174 -0
- package/dist/repo/shared_libs/play-runtime/static-pipeline-types.ts +147 -0
- package/dist/repo/shared_libs/play-runtime/suspension.ts +68 -0
- package/dist/repo/shared_libs/play-runtime/tool-batch-executor.ts +146 -0
- package/dist/repo/shared_libs/play-runtime/tool-result.ts +387 -0
- package/dist/repo/shared_libs/play-runtime/tracing.ts +31 -0
- package/dist/repo/shared_libs/play-runtime/waterfall-replay.ts +75 -0
- package/dist/repo/shared_libs/play-runtime/worker-api-types.ts +140 -0
- package/dist/repo/shared_libs/plays/artifact-transport.ts +14 -0
- package/dist/repo/shared_libs/plays/artifact-types.ts +49 -0
- package/dist/repo/shared_libs/plays/bundling/index.ts +1346 -0
- package/dist/repo/shared_libs/plays/compiler-manifest.ts +186 -0
- package/dist/repo/shared_libs/plays/contracts.ts +51 -0
- package/dist/repo/shared_libs/plays/dataset.ts +308 -0
- package/dist/repo/shared_libs/plays/definition.ts +264 -0
- package/dist/repo/shared_libs/plays/file-refs.ts +11 -0
- package/dist/repo/shared_libs/plays/rate-limit-scheduler.ts +206 -0
- package/dist/repo/shared_libs/plays/resolve-static-pipeline.ts +164 -0
- package/dist/repo/shared_libs/plays/row-identity.ts +302 -0
- package/dist/repo/shared_libs/plays/runtime-validation.ts +415 -0
- package/dist/repo/shared_libs/plays/static-pipeline.ts +560 -0
- package/dist/repo/shared_libs/temporal/constants.ts +39 -0
- package/dist/repo/shared_libs/temporal/preview-config.ts +153 -0
- package/package.json +14 -12
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
import { existsSync } from 'node:fs';
|
|
2
|
+
import * as path from 'node:path';
|
|
3
|
+
import * as dotenv from 'dotenv';
|
|
4
|
+
import {
|
|
5
|
+
LOCAL_TEMPORAL_ADDRESS,
|
|
6
|
+
LOCAL_TEMPORAL_NAMESPACE,
|
|
7
|
+
} from './constants';
|
|
8
|
+
|
|
9
|
+
export const PLAY_TASK_QUEUE = 'deepline-plays';
|
|
10
|
+
// Internal process flag set by `bun run dev:v2 --preview`. Do not put this in
|
|
11
|
+
// .env.local; the dev command owns the mode selection so local/preview runs are
|
|
12
|
+
// obvious at the call site.
|
|
13
|
+
export const DEEPLINE_V2_INFRA_MODE_ENV = 'DEEPLINE_V2_INFRA_MODE';
|
|
14
|
+
export const PREVIEW_PLAYS_ENV_FILE = '.env.plays.preview';
|
|
15
|
+
|
|
16
|
+
type TemporalTlsConfig = {
|
|
17
|
+
clientCertPair: {
|
|
18
|
+
crt: Uint8Array;
|
|
19
|
+
key: Uint8Array;
|
|
20
|
+
};
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
export type DeeplineTemporalConfig = {
|
|
24
|
+
address: string;
|
|
25
|
+
namespace: string;
|
|
26
|
+
taskQueue: string;
|
|
27
|
+
tls?: TemporalTlsConfig;
|
|
28
|
+
deployment: 'local' | 'preview';
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
let loadedPreviewEnv = false;
|
|
32
|
+
|
|
33
|
+
function decodePemEnv(value: string): Uint8Array {
|
|
34
|
+
return Buffer.from(value.replace(/\\n/g, '\n'), 'utf-8');
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export function isPreviewPlaysDeployment(): boolean {
|
|
38
|
+
const mode = process.env[DEEPLINE_V2_INFRA_MODE_ENV]?.trim().toLowerCase();
|
|
39
|
+
return mode === 'external' || mode === 'preview';
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export function isCloudflarePlaysDeployment(): boolean {
|
|
43
|
+
const mode = process.env[DEEPLINE_V2_INFRA_MODE_ENV]?.trim().toLowerCase();
|
|
44
|
+
return mode === 'cloudflare';
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* True when the play runtime callback baseUrl must be a public URL — i.e. when
|
|
49
|
+
* the plays runner runs outside the local app process and can't reach
|
|
50
|
+
* 127.0.0.1. Both `external` (Daytona) and `cloudflare` (Workers/DOs) need
|
|
51
|
+
* this; only fully-local mode can keep baseUrl on localhost.
|
|
52
|
+
*/
|
|
53
|
+
export function requiresPublicCallbackBaseUrl(): boolean {
|
|
54
|
+
return isPreviewPlaysDeployment() || isCloudflarePlaysDeployment();
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export function loadPreviewPlaysEnv(cwd = process.cwd()): void {
|
|
58
|
+
if (loadedPreviewEnv || !isPreviewPlaysDeployment()) {
|
|
59
|
+
return;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
const envPath = path.join(cwd, PREVIEW_PLAYS_ENV_FILE);
|
|
63
|
+
if (existsSync(envPath)) {
|
|
64
|
+
dotenv.config({ path: envPath, override: true });
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
loadedPreviewEnv = true;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
function temporalTlsFromEnv(): TemporalTlsConfig | undefined {
|
|
71
|
+
const cert =
|
|
72
|
+
process.env.TEMPORAL_TLS_CERT_DATA ?? process.env.FAS_TEMPORAL_CERT_DATA;
|
|
73
|
+
const key =
|
|
74
|
+
process.env.TEMPORAL_TLS_KEY_DATA ?? process.env.FAS_TEMPORAL_KEY_DATA;
|
|
75
|
+
|
|
76
|
+
if (!cert && !key) {
|
|
77
|
+
return undefined;
|
|
78
|
+
}
|
|
79
|
+
if (!cert || !key) {
|
|
80
|
+
throw new Error(
|
|
81
|
+
'Temporal Cloud mTLS requires both TEMPORAL_TLS_CERT_DATA and TEMPORAL_TLS_KEY_DATA.',
|
|
82
|
+
);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
return {
|
|
86
|
+
clientCertPair: {
|
|
87
|
+
crt: decodePemEnv(cert),
|
|
88
|
+
key: decodePemEnv(key),
|
|
89
|
+
},
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function temporalAddressFromEnv(): string | undefined {
|
|
94
|
+
return (
|
|
95
|
+
process.env.TEMPORAL_ADDRESS?.trim() ||
|
|
96
|
+
process.env.FAS_TEMPORAL_ADDRESS?.trim() ||
|
|
97
|
+
undefined
|
|
98
|
+
);
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
function temporalNamespaceFromEnv(address: string): string | undefined {
|
|
102
|
+
const previewConfigured = process.env.FAS_TEMPORAL_NAMESPACE?.trim();
|
|
103
|
+
if (previewConfigured) {
|
|
104
|
+
return previewConfigured;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
const host = address.split(':')[0] ?? '';
|
|
108
|
+
const temporalCloudNamespace = /^([^.]+\.[^.]+)\.tmprl\.cloud$/i.exec(host);
|
|
109
|
+
if (temporalCloudNamespace?.[1]) {
|
|
110
|
+
return temporalCloudNamespace[1];
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
const configured = process.env.TEMPORAL_NAMESPACE?.trim();
|
|
114
|
+
return configured || undefined;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
export function getDeeplineTemporalConfig(options?: {
|
|
118
|
+
loadPreviewEnv?: boolean;
|
|
119
|
+
}): DeeplineTemporalConfig {
|
|
120
|
+
if (options?.loadPreviewEnv !== false) {
|
|
121
|
+
loadPreviewPlaysEnv();
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
if (isPreviewPlaysDeployment()) {
|
|
125
|
+
const address = temporalAddressFromEnv();
|
|
126
|
+
const namespace = address ? temporalNamespaceFromEnv(address) : undefined;
|
|
127
|
+
if (!address) {
|
|
128
|
+
throw new Error(
|
|
129
|
+
'dev:v2 preview mode requires TEMPORAL_ADDRESS or FAS_TEMPORAL_ADDRESS.',
|
|
130
|
+
);
|
|
131
|
+
}
|
|
132
|
+
if (!namespace) {
|
|
133
|
+
throw new Error(
|
|
134
|
+
'dev:v2 preview mode requires TEMPORAL_NAMESPACE or FAS_TEMPORAL_NAMESPACE.',
|
|
135
|
+
);
|
|
136
|
+
}
|
|
137
|
+
return {
|
|
138
|
+
address,
|
|
139
|
+
namespace,
|
|
140
|
+
taskQueue: process.env.TEMPORAL_TASK_QUEUE?.trim() || PLAY_TASK_QUEUE,
|
|
141
|
+
tls: temporalTlsFromEnv(),
|
|
142
|
+
deployment: 'preview',
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
return {
|
|
147
|
+
address: process.env.TEMPORAL_ADDRESS ?? LOCAL_TEMPORAL_ADDRESS,
|
|
148
|
+
namespace: process.env.TEMPORAL_NAMESPACE ?? LOCAL_TEMPORAL_NAMESPACE,
|
|
149
|
+
taskQueue: process.env.TEMPORAL_TASK_QUEUE?.trim() || PLAY_TASK_QUEUE,
|
|
150
|
+
tls: temporalTlsFromEnv(),
|
|
151
|
+
deployment: 'local',
|
|
152
|
+
};
|
|
153
|
+
}
|
package/package.json
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "deepline",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.1.1",
|
|
4
4
|
"description": "Deepline SDK + CLI — B2B data enrichment powered by durable cloud execution",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
7
7
|
"type": "git",
|
|
8
|
-
"url": "https://github.com/deeplineai/deepline"
|
|
8
|
+
"url": "git+https://github.com/deeplineai/deepline.git"
|
|
9
9
|
},
|
|
10
10
|
"homepage": "https://code.deepline.com",
|
|
11
11
|
"keywords": [
|
|
@@ -17,17 +17,17 @@
|
|
|
17
17
|
"cli",
|
|
18
18
|
"sdk"
|
|
19
19
|
],
|
|
20
|
-
"main": "./dist/index.
|
|
21
|
-
"module": "./dist/index.
|
|
20
|
+
"main": "./dist/index.js",
|
|
21
|
+
"module": "./dist/index.mjs",
|
|
22
22
|
"types": "./dist/index.d.ts",
|
|
23
23
|
"bin": {
|
|
24
|
-
"deepline": "
|
|
24
|
+
"deepline": "dist/cli/index.mjs"
|
|
25
25
|
},
|
|
26
26
|
"exports": {
|
|
27
27
|
".": {
|
|
28
28
|
"types": "./dist/index.d.ts",
|
|
29
|
-
"import": "./dist/index.
|
|
30
|
-
"require": "./dist/index.
|
|
29
|
+
"import": "./dist/index.mjs",
|
|
30
|
+
"require": "./dist/index.js"
|
|
31
31
|
}
|
|
32
32
|
},
|
|
33
33
|
"files": [
|
|
@@ -39,16 +39,18 @@
|
|
|
39
39
|
"scripts": {
|
|
40
40
|
"build": "tsup",
|
|
41
41
|
"dev": "tsup --watch",
|
|
42
|
-
"typecheck": "tsc --noEmit",
|
|
42
|
+
"typecheck": "tsc --noEmit --declaration false --declarationMap false",
|
|
43
43
|
"prepublishOnly": "tsup"
|
|
44
44
|
},
|
|
45
45
|
"dependencies": {
|
|
46
|
+
"commander": "^14.0.3",
|
|
46
47
|
"csv-parse": "^5.6.0",
|
|
47
|
-
"csv-stringify": "^6.5.0"
|
|
48
|
+
"csv-stringify": "^6.5.0",
|
|
49
|
+
"esbuild": "^0.25.11",
|
|
50
|
+
"typescript": "^5.9.3",
|
|
51
|
+
"@types/node": "^20.0.0"
|
|
48
52
|
},
|
|
49
53
|
"devDependencies": {
|
|
50
|
-
"tsup": "^8.0.0"
|
|
51
|
-
"typescript": "^5.4.0",
|
|
52
|
-
"@types/node": "^20.0.0"
|
|
54
|
+
"tsup": "^8.0.0"
|
|
53
55
|
}
|
|
54
56
|
}
|