deepline 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/index.js +212 -54
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/index.mjs +198 -40
- package/dist/cli/index.mjs.map +1 -1
- package/dist/index.d.mts +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/index.mjs +1 -1
- package/dist/repo/apps/play-runner-workers/src/coordinator-entry.ts +3256 -0
- package/dist/repo/apps/play-runner-workers/src/dedup-do.ts +710 -0
- package/dist/repo/apps/play-runner-workers/src/entry.ts +5070 -0
- package/dist/repo/apps/play-runner-workers/src/runtime/README.md +21 -0
- package/dist/repo/apps/play-runner-workers/src/runtime/batching.ts +177 -0
- package/dist/repo/apps/play-runner-workers/src/runtime/execution-plan.ts +52 -0
- package/dist/repo/apps/play-runner-workers/src/runtime/tool-batch.ts +100 -0
- package/dist/repo/apps/play-runner-workers/src/runtime/tool-result.ts +184 -0
- package/dist/repo/sdk/src/cli/commands/auth.ts +482 -0
- package/dist/repo/sdk/src/cli/commands/billing.ts +188 -0
- package/dist/repo/sdk/src/cli/commands/csv.ts +123 -0
- package/dist/repo/sdk/src/cli/commands/db.ts +119 -0
- package/dist/repo/sdk/src/cli/commands/feedback.ts +40 -0
- package/dist/repo/sdk/src/cli/commands/org.ts +117 -0
- package/dist/repo/sdk/src/cli/commands/play.ts +3200 -0
- package/dist/repo/sdk/src/cli/commands/tools.ts +687 -0
- package/dist/repo/sdk/src/cli/dataset-stats.ts +341 -0
- package/dist/repo/sdk/src/cli/index.ts +138 -0
- package/dist/repo/sdk/src/cli/progress.ts +135 -0
- package/dist/repo/sdk/src/cli/trace.ts +61 -0
- package/dist/repo/sdk/src/cli/utils.ts +145 -0
- package/dist/repo/sdk/src/client.ts +1188 -0
- package/dist/repo/sdk/src/compat.ts +77 -0
- package/dist/repo/sdk/src/config.ts +285 -0
- package/dist/repo/sdk/src/errors.ts +125 -0
- package/dist/repo/sdk/src/http.ts +391 -0
- package/dist/repo/sdk/src/index.ts +139 -0
- package/dist/repo/sdk/src/play.ts +1330 -0
- package/dist/repo/sdk/src/plays/bundle-play-file.ts +133 -0
- package/dist/repo/sdk/src/plays/harness-stub.ts +210 -0
- package/dist/repo/sdk/src/plays/local-file-discovery.ts +326 -0
- package/dist/repo/sdk/src/tool-output.ts +489 -0
- package/dist/repo/sdk/src/types.ts +669 -0
- package/dist/repo/sdk/src/version.ts +2 -0
- package/dist/repo/sdk/src/worker-play-entry.ts +286 -0
- package/dist/repo/shared_libs/observability/node-tracing.ts +129 -0
- package/dist/repo/shared_libs/observability/tracing.ts +98 -0
- package/dist/repo/shared_libs/play-runtime/backend.ts +139 -0
- package/dist/repo/shared_libs/play-runtime/batch-runtime.ts +182 -0
- package/dist/repo/shared_libs/play-runtime/batching-types.ts +91 -0
- package/dist/repo/shared_libs/play-runtime/context.ts +3999 -0
- package/dist/repo/shared_libs/play-runtime/coordinator-headers.ts +78 -0
- package/dist/repo/shared_libs/play-runtime/ctx-contract.ts +250 -0
- package/dist/repo/shared_libs/play-runtime/ctx-types.ts +713 -0
- package/dist/repo/shared_libs/play-runtime/dataset-id.ts +10 -0
- package/dist/repo/shared_libs/play-runtime/db-session-crypto.ts +304 -0
- package/dist/repo/shared_libs/play-runtime/db-session.ts +462 -0
- package/dist/repo/shared_libs/play-runtime/dedup-backend.ts +0 -0
- package/dist/repo/shared_libs/play-runtime/default-batch-strategies.ts +124 -0
- package/dist/repo/shared_libs/play-runtime/execution-plan.ts +262 -0
- package/dist/repo/shared_libs/play-runtime/live-events.ts +214 -0
- package/dist/repo/shared_libs/play-runtime/live-state-contract.ts +50 -0
- package/dist/repo/shared_libs/play-runtime/map-execution-frame.ts +114 -0
- package/dist/repo/shared_libs/play-runtime/map-row-identity.ts +158 -0
- package/dist/repo/shared_libs/play-runtime/profiles.ts +90 -0
- package/dist/repo/shared_libs/play-runtime/progress-emitter.ts +172 -0
- package/dist/repo/shared_libs/play-runtime/protocol.ts +121 -0
- package/dist/repo/shared_libs/play-runtime/public-play-contract.ts +42 -0
- package/dist/repo/shared_libs/play-runtime/result-normalization.ts +33 -0
- package/dist/repo/shared_libs/play-runtime/runtime-actions.ts +208 -0
- package/dist/repo/shared_libs/play-runtime/runtime-api.ts +1873 -0
- package/dist/repo/shared_libs/play-runtime/runtime-constraints.ts +2 -0
- package/dist/repo/shared_libs/play-runtime/runtime-pg-driver-neon-serverless.ts +201 -0
- package/dist/repo/shared_libs/play-runtime/runtime-pg-driver-pg.ts +48 -0
- package/dist/repo/shared_libs/play-runtime/runtime-pg-driver.ts +84 -0
- package/dist/repo/shared_libs/play-runtime/scheduler-backend.ts +174 -0
- package/dist/repo/shared_libs/play-runtime/static-pipeline-types.ts +147 -0
- package/dist/repo/shared_libs/play-runtime/suspension.ts +68 -0
- package/dist/repo/shared_libs/play-runtime/tool-batch-executor.ts +146 -0
- package/dist/repo/shared_libs/play-runtime/tool-result.ts +387 -0
- package/dist/repo/shared_libs/play-runtime/tracing.ts +31 -0
- package/dist/repo/shared_libs/play-runtime/waterfall-replay.ts +75 -0
- package/dist/repo/shared_libs/play-runtime/worker-api-types.ts +140 -0
- package/dist/repo/shared_libs/plays/artifact-transport.ts +14 -0
- package/dist/repo/shared_libs/plays/artifact-types.ts +49 -0
- package/dist/repo/shared_libs/plays/bundling/index.ts +1346 -0
- package/dist/repo/shared_libs/plays/compiler-manifest.ts +186 -0
- package/dist/repo/shared_libs/plays/contracts.ts +51 -0
- package/dist/repo/shared_libs/plays/dataset.ts +308 -0
- package/dist/repo/shared_libs/plays/definition.ts +264 -0
- package/dist/repo/shared_libs/plays/file-refs.ts +11 -0
- package/dist/repo/shared_libs/plays/rate-limit-scheduler.ts +206 -0
- package/dist/repo/shared_libs/plays/resolve-static-pipeline.ts +164 -0
- package/dist/repo/shared_libs/plays/row-identity.ts +302 -0
- package/dist/repo/shared_libs/plays/runtime-validation.ts +415 -0
- package/dist/repo/shared_libs/plays/static-pipeline.ts +560 -0
- package/dist/repo/shared_libs/temporal/constants.ts +39 -0
- package/dist/repo/shared_libs/temporal/preview-config.ts +153 -0
- package/package.json +4 -4
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
import { tmpdir } from 'node:os';
|
|
2
|
+
import { dirname, join, resolve } from 'node:path';
|
|
3
|
+
import { fileURLToPath } from 'node:url';
|
|
4
|
+
import { existsSync } from 'node:fs';
|
|
5
|
+
import {
|
|
6
|
+
bundlePlayFile as bundlePlayFileCore,
|
|
7
|
+
type BundlePlayFileOptions,
|
|
8
|
+
type BundledPlayFileSuccess,
|
|
9
|
+
type BundledPlayFileResult,
|
|
10
|
+
type ImportedPlayDependency,
|
|
11
|
+
type PlayBundlingAdapter,
|
|
12
|
+
type PlayLocalFileDiscoveryError,
|
|
13
|
+
type PlayLocalFileReference,
|
|
14
|
+
} from '../../../shared_libs/plays/bundling/index.js';
|
|
15
|
+
import {
|
|
16
|
+
PLAY_ARTIFACT_KINDS,
|
|
17
|
+
PLAY_BACKEND_DESCRIPTORS,
|
|
18
|
+
type PlayArtifactKind,
|
|
19
|
+
} from '../../../shared_libs/play-runtime/backend.js';
|
|
20
|
+
import { resolveExecutionProfile } from '../../../shared_libs/play-runtime/profiles.js';
|
|
21
|
+
import {
|
|
22
|
+
discoverPackagedLocalFiles,
|
|
23
|
+
} from './local-file-discovery.js';
|
|
24
|
+
|
|
25
|
+
export type {
|
|
26
|
+
BundlePlayFileOptions,
|
|
27
|
+
BundledPlayFileSuccess,
|
|
28
|
+
BundledPlayFileResult,
|
|
29
|
+
ImportedPlayDependency,
|
|
30
|
+
PlayLocalFileDiscoveryError,
|
|
31
|
+
PlayLocalFileReference,
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
export type {
|
|
35
|
+
PlayArtifactCompatibility,
|
|
36
|
+
PlayBundleArtifact,
|
|
37
|
+
PlayImportPolicy,
|
|
38
|
+
PlayPackageImport,
|
|
39
|
+
PlayRuntimeFeature,
|
|
40
|
+
} from '../../../shared_libs/plays/bundling/index.js';
|
|
41
|
+
|
|
42
|
+
export {
|
|
43
|
+
extractDefinedPlayName,
|
|
44
|
+
} from '../../../shared_libs/plays/bundling/index.js';
|
|
45
|
+
|
|
46
|
+
const PLAY_BUNDLE_CACHE_VERSION = 24;
|
|
47
|
+
const MODULE_DIR = dirname(fileURLToPath(import.meta.url));
|
|
48
|
+
const SDK_PACKAGE_ROOT = resolve(MODULE_DIR, '..', '..');
|
|
49
|
+
const SOURCE_REPO_ROOT = resolve(SDK_PACKAGE_ROOT, '..');
|
|
50
|
+
const HAS_SOURCE_BUNDLING_SOURCES = existsSync(
|
|
51
|
+
resolve(SOURCE_REPO_ROOT, 'apps', 'play-runner-workers', 'src', 'entry.ts'),
|
|
52
|
+
);
|
|
53
|
+
const PACKAGED_REPO_ROOT = resolve(SDK_PACKAGE_ROOT, 'dist', 'repo');
|
|
54
|
+
const HAS_PACKAGED_BUNDLING_SOURCES = existsSync(
|
|
55
|
+
resolve(PACKAGED_REPO_ROOT, 'apps', 'play-runner-workers', 'src', 'entry.ts'),
|
|
56
|
+
);
|
|
57
|
+
const PROJECT_ROOT = HAS_SOURCE_BUNDLING_SOURCES
|
|
58
|
+
? SOURCE_REPO_ROOT
|
|
59
|
+
: HAS_PACKAGED_BUNDLING_SOURCES
|
|
60
|
+
? PACKAGED_REPO_ROOT
|
|
61
|
+
: resolve(SDK_PACKAGE_ROOT, '..');
|
|
62
|
+
const SDK_SOURCE_ROOT = HAS_SOURCE_BUNDLING_SOURCES
|
|
63
|
+
? resolve(SOURCE_REPO_ROOT, 'sdk', 'src')
|
|
64
|
+
: HAS_PACKAGED_BUNDLING_SOURCES
|
|
65
|
+
? resolve(PACKAGED_REPO_ROOT, 'sdk', 'src')
|
|
66
|
+
: resolve(SDK_PACKAGE_ROOT, 'src');
|
|
67
|
+
const SDK_PACKAGE_JSON = resolve(SDK_PACKAGE_ROOT, 'package.json');
|
|
68
|
+
const SDK_ENTRY_FILE = resolve(SDK_SOURCE_ROOT, 'index.ts');
|
|
69
|
+
const SDK_TYPES_ENTRY_FILE = resolve(SDK_PACKAGE_ROOT, 'dist', 'index.d.ts');
|
|
70
|
+
const SDK_WORKERS_ENTRY_FILE = resolve(SDK_SOURCE_ROOT, 'worker-play-entry.ts');
|
|
71
|
+
const WORKERS_HARNESS_ENTRY_FILE = resolve(PROJECT_ROOT, 'apps', 'play-runner-workers', 'src', 'entry.ts');
|
|
72
|
+
const WORKERS_HARNESS_FILES_DIR = resolve(PROJECT_ROOT, 'apps', 'play-runner-workers', 'src');
|
|
73
|
+
|
|
74
|
+
let hasWarnedAboutNonDevelopmentBundling = false;
|
|
75
|
+
|
|
76
|
+
function warnAboutNonDevelopmentBundling(filePath: string): void {
|
|
77
|
+
if (hasWarnedAboutNonDevelopmentBundling) {
|
|
78
|
+
return;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
const nodeEnv = String(process.env.NODE_ENV ?? '').trim().toLowerCase();
|
|
82
|
+
if (!nodeEnv || nodeEnv === 'development' || nodeEnv === 'test') {
|
|
83
|
+
return;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
hasWarnedAboutNonDevelopmentBundling = true;
|
|
87
|
+
console.warn(
|
|
88
|
+
`[deepline] Warning: live play bundling was invoked while NODE_ENV=${nodeEnv} for ${filePath}. ` +
|
|
89
|
+
'This source-first SDK path is intended for local development. ' +
|
|
90
|
+
'For preview/production, run a published or prebuilt play reference instead of bundling source at runtime.',
|
|
91
|
+
);
|
|
92
|
+
console.warn(
|
|
93
|
+
'[deepline] Preferred production call pattern: client.play("person-to-email").run(...) ' +
|
|
94
|
+
'or run a previously registered/published org play.',
|
|
95
|
+
);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function defaultPlayBundleTarget(): PlayArtifactKind {
|
|
99
|
+
return PLAY_BACKEND_DESCRIPTORS[
|
|
100
|
+
resolveExecutionProfile(null).runner
|
|
101
|
+
].artifactKind;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
export function createSdkPlayBundlingAdapter(): PlayBundlingAdapter {
|
|
105
|
+
return {
|
|
106
|
+
projectRoot: PROJECT_ROOT,
|
|
107
|
+
nodeModulesDir: resolve(PROJECT_ROOT, 'node_modules'),
|
|
108
|
+
cacheDir: join(tmpdir(), `deepline-play-artifacts-v${PLAY_BUNDLE_CACHE_VERSION}`),
|
|
109
|
+
sdkSourceRoot: SDK_SOURCE_ROOT,
|
|
110
|
+
sdkPackageJson: SDK_PACKAGE_JSON,
|
|
111
|
+
sdkEntryFile: SDK_ENTRY_FILE,
|
|
112
|
+
sdkTypesEntryFile: existsSync(SDK_TYPES_ENTRY_FILE)
|
|
113
|
+
? SDK_TYPES_ENTRY_FILE
|
|
114
|
+
: SDK_ENTRY_FILE,
|
|
115
|
+
sdkWorkersEntryFile: SDK_WORKERS_ENTRY_FILE,
|
|
116
|
+
workersHarnessEntryFile: WORKERS_HARNESS_ENTRY_FILE,
|
|
117
|
+
workersHarnessFilesDir: WORKERS_HARNESS_FILES_DIR,
|
|
118
|
+
discoverPackagedLocalFiles,
|
|
119
|
+
warnAboutNonDevelopmentBundling,
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
export async function bundlePlayFile(
|
|
124
|
+
filePath: string,
|
|
125
|
+
options: BundlePlayFileOptions = {},
|
|
126
|
+
): Promise<BundledPlayFileResult> {
|
|
127
|
+
return bundlePlayFileCore(filePath, {
|
|
128
|
+
target: options.target ?? defaultPlayBundleTarget(),
|
|
129
|
+
adapter: createSdkPlayBundlingAdapter(),
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
export { PLAY_ARTIFACT_KINDS };
|
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-play stub that calls into the long-lived Play Harness Worker via
|
|
3
|
+
* `env.HARNESS`.
|
|
4
|
+
*
|
|
5
|
+
* This file is what ends up bundled into every per-play Worker. It MUST
|
|
6
|
+
* stay tiny — every byte here is paid by every fresh-graphHash V8
|
|
7
|
+
* compile in CF's WorkerLoader. Target: <2 KB minified.
|
|
8
|
+
*
|
|
9
|
+
* What it does:
|
|
10
|
+
* - Exposes thin functions that look like the in-bundle implementations
|
|
11
|
+
* they're replacing (validate, runtime-api call, …).
|
|
12
|
+
* - Each function calls `env.HARNESS.<method>(...)` — the typed RPC
|
|
13
|
+
* stub provided by the Cloudflare service binding.
|
|
14
|
+
*
|
|
15
|
+
* What it does NOT do:
|
|
16
|
+
* - Does NOT import zod (that's the whole point — zod stays in the
|
|
17
|
+
* harness Worker).
|
|
18
|
+
* - Does NOT import the harness Worker's `PlayHarness` class (that
|
|
19
|
+
* would re-bundle the whole harness; only TYPES are imported).
|
|
20
|
+
* - Does NOT cache RPC results across plays — caching belongs to
|
|
21
|
+
* callers if they want it. This file stays stateless so it composes
|
|
22
|
+
* cleanly under the per-play orchestrator's expectations.
|
|
23
|
+
*
|
|
24
|
+
* How the env binding reaches per-play code:
|
|
25
|
+
* The coordinator's `WorkerLoader` factory passes `HARNESS` in the
|
|
26
|
+
* per-play Worker's `env` map (see
|
|
27
|
+
* `apps/play-runner-workers/src/coordinator-entry.ts` near
|
|
28
|
+
* `loadDynamicPlayWorkerSync`). The per-play harness then reads it
|
|
29
|
+
* off `WorkerEnv` and stashes it via `setHarnessBinding`.
|
|
30
|
+
*
|
|
31
|
+
* @see {@link ../../../apps/play-harness-worker/README.md}
|
|
32
|
+
*/
|
|
33
|
+
|
|
34
|
+
import type {
|
|
35
|
+
PlayHarnessRpc,
|
|
36
|
+
RuntimeApiCallInput,
|
|
37
|
+
RuntimeApiCallResult,
|
|
38
|
+
RuntimePayloadSchemaId,
|
|
39
|
+
ValidatePayloadResult,
|
|
40
|
+
} from '../../../apps/play-harness-worker/src/rpc-types';
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Service-binding RPC stub shape — what `env.HARNESS` looks like inside
|
|
44
|
+
* a per-play Worker. Cloudflare exposes WorkerEntrypoint methods as
|
|
45
|
+
* promise-returning RPC stubs; the type matches `PlayHarnessRpc`.
|
|
46
|
+
*/
|
|
47
|
+
export type HarnessBinding = PlayHarnessRpc;
|
|
48
|
+
type HarnessFetchBinding = HarnessBinding & {
|
|
49
|
+
fetch(input: RequestInfo, init?: RequestInit): Promise<Response>;
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Module-level holder for the binding. Set by the per-play harness on
|
|
54
|
+
* worker init (see `apps/play-runner-workers/src/entry.ts → setHarnessBinding`).
|
|
55
|
+
*
|
|
56
|
+
* Why a module-level holder instead of threading the binding through
|
|
57
|
+
* every call site: most leaf-call sites are deep inside SDK utilities
|
|
58
|
+
* (`ctx.tool`, `ctx.csv`, etc.) and threading a binding everywhere would
|
|
59
|
+
* be a structural mess. A single set-once holder, set very early in the
|
|
60
|
+
* worker's lifecycle, gives us clean call sites without sacrificing
|
|
61
|
+
* isolation — each per-play Worker is its own isolate, so the holder is
|
|
62
|
+
* never shared across plays.
|
|
63
|
+
*/
|
|
64
|
+
let cachedBinding: HarnessBinding | null = null;
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Install the harness binding for this worker. Called once on init from
|
|
68
|
+
* the per-play harness. Subsequent calls overwrite — the last writer
|
|
69
|
+
* wins. In practice the per-play harness sets this exactly once before
|
|
70
|
+
* any user code runs, so overwrites should never happen in production.
|
|
71
|
+
*/
|
|
72
|
+
export function setHarnessBinding(binding: HarnessBinding | null): void {
|
|
73
|
+
cachedBinding = binding;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Read the installed binding or throw a loud error. We deliberately
|
|
78
|
+
* throw rather than fall back to a HTTP-fetch path: fall-backs hide
|
|
79
|
+
* misconfiguration and let perf regressions creep in silently. If
|
|
80
|
+
* HARNESS is missing, the coordinator forgot to wire it in the
|
|
81
|
+
* WorkerLoader factory — that's a deploy bug, not a runtime contingency.
|
|
82
|
+
*/
|
|
83
|
+
function requireBinding(): HarnessBinding {
|
|
84
|
+
if (!cachedBinding) {
|
|
85
|
+
throw new Error(
|
|
86
|
+
'[harness-stub] env.HARNESS is not wired. The coordinator must pass HARNESS in the WorkerLoader factory env. ' +
|
|
87
|
+
'See apps/play-runner-workers/src/coordinator-entry.ts → loadDynamicPlayWorkerSync.',
|
|
88
|
+
);
|
|
89
|
+
}
|
|
90
|
+
return cachedBinding;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// ---------------------------------------------------------------------------
|
|
94
|
+
// Public stub functions — these are what call sites use instead of the
|
|
95
|
+
// in-bundle implementations they replace.
|
|
96
|
+
//
|
|
97
|
+
// Each function MUST stay a one-line forwarder to keep the per-play
|
|
98
|
+
// bundle minimal. If a stub starts growing logic, that logic belongs in
|
|
99
|
+
// the harness Worker, not here.
|
|
100
|
+
// ---------------------------------------------------------------------------
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Liveness ping. Mostly used by tests + diagnostics that run inside a
|
|
104
|
+
* play context; production code rarely calls this.
|
|
105
|
+
*/
|
|
106
|
+
export async function harnessPing(): Promise<{ ok: true; ts: number }> {
|
|
107
|
+
return requireBinding().ping();
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Forward a runtime-API call through the harness.
|
|
112
|
+
*
|
|
113
|
+
* Replaces what was previously a direct `fetch(env.RUNTIME_API, ...)`
|
|
114
|
+
* call in the per-play harness. The transport is identical (HTTPS to
|
|
115
|
+
* the upstream), but the HTTP client lives in the harness Worker so
|
|
116
|
+
* its retry/timeout/error code doesn't ride in every per-play bundle.
|
|
117
|
+
*/
|
|
118
|
+
export async function harnessRuntimeApiCall(
|
|
119
|
+
input: RuntimeApiCallInput,
|
|
120
|
+
): Promise<RuntimeApiCallResult> {
|
|
121
|
+
return requireBinding().runtimeApiCall(input);
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* Start or continue a map dataset write inside the harness, so this call
|
|
126
|
+
* skips per-play callback overhead for heavy Postgres writes.
|
|
127
|
+
*/
|
|
128
|
+
export async function harnessStartSheetDataset(input: {
|
|
129
|
+
baseUrl: string;
|
|
130
|
+
executorToken: string;
|
|
131
|
+
playName: string;
|
|
132
|
+
tableNamespace: string;
|
|
133
|
+
sheetContract: unknown;
|
|
134
|
+
rows: Array<Record<string, unknown>>;
|
|
135
|
+
runId: string;
|
|
136
|
+
userEmail?: string | null;
|
|
137
|
+
}): Promise<{
|
|
138
|
+
inserted: number;
|
|
139
|
+
skipped: number;
|
|
140
|
+
pendingRows: Array<Record<string, unknown>>;
|
|
141
|
+
completedRows: Array<Record<string, unknown>>;
|
|
142
|
+
tableNamespace: string;
|
|
143
|
+
}> {
|
|
144
|
+
return requireBinding().startSheetDataset(input);
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* Persist completed map rows inside the harness, directly through the
|
|
149
|
+
* scoped Neon callback surface.
|
|
150
|
+
*/
|
|
151
|
+
export async function harnessPersistCompletedSheetRows(input: {
|
|
152
|
+
baseUrl: string;
|
|
153
|
+
executorToken: string;
|
|
154
|
+
playName: string;
|
|
155
|
+
tableNamespace: string;
|
|
156
|
+
sheetContract: unknown;
|
|
157
|
+
rows: Array<Record<string, unknown>>;
|
|
158
|
+
outputFields: string[];
|
|
159
|
+
runId: string;
|
|
160
|
+
userEmail?: string | null;
|
|
161
|
+
}): Promise<{ ok: true; rowsWritten: number; tableNamespace: string }> {
|
|
162
|
+
return requireBinding().persistCompletedMapRows(input);
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* Stream a staged R2 object through the harness Worker. Unlike the legacy
|
|
167
|
+
* signed-URL helper this never touches the Vercel runtime route, and it keeps
|
|
168
|
+
* large CSV bodies as streams instead of serializing them through RPC.
|
|
169
|
+
*/
|
|
170
|
+
export async function harnessFetchStagedFile(input: {
|
|
171
|
+
executorToken: string;
|
|
172
|
+
storageKey: string;
|
|
173
|
+
method?: 'GET' | 'HEAD';
|
|
174
|
+
range?: { offset: number; length: number };
|
|
175
|
+
}): Promise<Response> {
|
|
176
|
+
const binding = requireBinding() as HarnessFetchBinding;
|
|
177
|
+
if (typeof binding.fetch !== 'function') {
|
|
178
|
+
throw new Error(
|
|
179
|
+
'[harness-stub] env.HARNESS does not expose fetch(); cannot stream staged R2 files through the harness.',
|
|
180
|
+
);
|
|
181
|
+
}
|
|
182
|
+
const url = new URL('https://play-harness.internal/r2/staged-file');
|
|
183
|
+
url.searchParams.set('storageKey', input.storageKey);
|
|
184
|
+
const headers: Record<string, string> = {
|
|
185
|
+
authorization: `Bearer ${input.executorToken}`,
|
|
186
|
+
};
|
|
187
|
+
if (input.range) {
|
|
188
|
+
const end = input.range.offset + input.range.length - 1;
|
|
189
|
+
headers.range = `bytes=${input.range.offset}-${end}`;
|
|
190
|
+
}
|
|
191
|
+
return binding.fetch(url.toString(), {
|
|
192
|
+
method: input.method ?? 'GET',
|
|
193
|
+
headers,
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/**
|
|
198
|
+
* Validate a payload against a named schema. The schema definitions
|
|
199
|
+
* (and zod itself) live in the harness Worker — see
|
|
200
|
+
* `apps/play-harness-worker/src/leaves/validate.ts → KNOWN_SCHEMAS`.
|
|
201
|
+
*
|
|
202
|
+
* Use a schema id of the form `tool:<provider>:<operation>` for tool
|
|
203
|
+
* inputs, or `runtime-api:<action>` for runtime-api request bodies.
|
|
204
|
+
*/
|
|
205
|
+
export async function harnessValidatePayload(
|
|
206
|
+
schemaId: RuntimePayloadSchemaId,
|
|
207
|
+
payload: unknown,
|
|
208
|
+
): Promise<ValidatePayloadResult> {
|
|
209
|
+
return requireBinding().validatePayload({ schemaId, payload });
|
|
210
|
+
}
|
|
@@ -0,0 +1,326 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { readFile, stat } from 'node:fs/promises';
|
|
3
|
+
import { basename, dirname, extname, isAbsolute, join, relative, resolve } from 'node:path';
|
|
4
|
+
import ts from 'typescript';
|
|
5
|
+
|
|
6
|
+
export interface PlayLocalFileReference {
|
|
7
|
+
sourceFragment: string;
|
|
8
|
+
logicalPath: string;
|
|
9
|
+
absolutePath: string;
|
|
10
|
+
bytes: number;
|
|
11
|
+
contentHash: string;
|
|
12
|
+
contentType: string;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export interface PlayLocalFileDiscoveryError {
|
|
16
|
+
sourceFragment: string;
|
|
17
|
+
message: string;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export interface PlayLocalFileDiscoveryResult {
|
|
21
|
+
files: PlayLocalFileReference[];
|
|
22
|
+
unresolved: PlayLocalFileDiscoveryError[];
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export interface PlayStagedFileRef {
|
|
26
|
+
storageKind: 'r2';
|
|
27
|
+
storageKey: string;
|
|
28
|
+
logicalPath: string;
|
|
29
|
+
fileName: string;
|
|
30
|
+
contentHash: string;
|
|
31
|
+
contentType: string;
|
|
32
|
+
bytes: number;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
type ConstMap = Map<string, string>;
|
|
36
|
+
|
|
37
|
+
const SOURCE_EXTENSIONS = ['.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs', '.json'];
|
|
38
|
+
|
|
39
|
+
function sha256(buffer: Buffer): string {
|
|
40
|
+
return createHash('sha256').update(buffer).digest('hex');
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function contentTypeForFile(filePath: string): string {
|
|
44
|
+
const extension = extname(filePath).toLowerCase();
|
|
45
|
+
if (extension === '.csv') return 'text/csv';
|
|
46
|
+
if (extension === '.json') return 'application/json';
|
|
47
|
+
if (extension === '.txt') return 'text/plain';
|
|
48
|
+
return 'application/octet-stream';
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function isCtxCsvCall(node: ts.CallExpression): boolean {
|
|
52
|
+
if (!ts.isPropertyAccessExpression(node.expression)) {
|
|
53
|
+
return false;
|
|
54
|
+
}
|
|
55
|
+
const target = node.expression.expression;
|
|
56
|
+
return (
|
|
57
|
+
ts.isIdentifier(target) &&
|
|
58
|
+
(target.text === 'ctx' || target.text.endsWith('Ctx')) &&
|
|
59
|
+
node.expression.name.text === 'csv'
|
|
60
|
+
);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function extractSourceFragment(source: string, node: ts.Node): string {
|
|
64
|
+
return source.slice(node.getStart(), node.getEnd()).trim();
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function referencesInputIdentifier(node: ts.Node): boolean {
|
|
68
|
+
if (ts.isIdentifier(node) && node.text === 'input') {
|
|
69
|
+
return true;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
return node.getChildren().some((child) => referencesInputIdentifier(child));
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function isRuntimeInputExpression(node: ts.Expression): boolean {
|
|
76
|
+
if (ts.isPropertyAccessExpression(node)) {
|
|
77
|
+
return ts.isIdentifier(node.expression) && node.expression.text === 'input';
|
|
78
|
+
}
|
|
79
|
+
if (ts.isElementAccessExpression(node)) {
|
|
80
|
+
return ts.isIdentifier(node.expression) && node.expression.text === 'input';
|
|
81
|
+
}
|
|
82
|
+
if (ts.isIdentifier(node)) {
|
|
83
|
+
return node.text === 'input';
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
if (ts.isParenthesizedExpression(node)) {
|
|
87
|
+
return isRuntimeInputExpression(node.expression);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
if (
|
|
91
|
+
ts.isBinaryExpression(node) &&
|
|
92
|
+
(node.operatorToken.kind === ts.SyntaxKind.QuestionQuestionToken ||
|
|
93
|
+
node.operatorToken.kind === ts.SyntaxKind.BarBarToken)
|
|
94
|
+
) {
|
|
95
|
+
return isRuntimeInputExpression(node.left) || isRuntimeInputExpression(node.right);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
if (ts.isConditionalExpression(node)) {
|
|
99
|
+
return (
|
|
100
|
+
isRuntimeInputExpression(node.condition) ||
|
|
101
|
+
isRuntimeInputExpression(node.whenTrue) ||
|
|
102
|
+
isRuntimeInputExpression(node.whenFalse)
|
|
103
|
+
);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
return referencesInputIdentifier(node);
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
function resolveStringExpression(node: ts.Expression, constants: ConstMap): string | null {
|
|
110
|
+
if (ts.isStringLiteralLike(node) || ts.isNoSubstitutionTemplateLiteral(node)) {
|
|
111
|
+
return node.text;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
if (ts.isParenthesizedExpression(node)) {
|
|
115
|
+
return resolveStringExpression(node.expression, constants);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
if (ts.isIdentifier(node)) {
|
|
119
|
+
return constants.get(node.text) ?? null;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
if (ts.isTemplateExpression(node)) {
|
|
123
|
+
let value = node.head.text;
|
|
124
|
+
for (const span of node.templateSpans) {
|
|
125
|
+
const resolved = resolveStringExpression(span.expression, constants);
|
|
126
|
+
if (resolved == null) {
|
|
127
|
+
return null;
|
|
128
|
+
}
|
|
129
|
+
value += resolved + span.literal.text;
|
|
130
|
+
}
|
|
131
|
+
return value;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
if (ts.isBinaryExpression(node) && node.operatorToken.kind === ts.SyntaxKind.PlusToken) {
|
|
135
|
+
const left = resolveStringExpression(node.left, constants);
|
|
136
|
+
const right = resolveStringExpression(node.right, constants);
|
|
137
|
+
if (left == null || right == null) {
|
|
138
|
+
return null;
|
|
139
|
+
}
|
|
140
|
+
return left + right;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
return null;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
function collectTopLevelStringConstants(sourceFile: ts.SourceFile): ConstMap {
|
|
147
|
+
const constants: ConstMap = new Map();
|
|
148
|
+
|
|
149
|
+
for (const statement of sourceFile.statements) {
|
|
150
|
+
if (!ts.isVariableStatement(statement)) {
|
|
151
|
+
continue;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
if (!(statement.declarationList.flags & ts.NodeFlags.Const)) {
|
|
155
|
+
continue;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
for (const declaration of statement.declarationList.declarations) {
|
|
159
|
+
if (!ts.isIdentifier(declaration.name) || !declaration.initializer) {
|
|
160
|
+
continue;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
const resolved = resolveStringExpression(declaration.initializer, constants);
|
|
164
|
+
if (resolved != null) {
|
|
165
|
+
constants.set(declaration.name.text, resolved);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
return constants;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
async function fileExists(filePath: string): Promise<boolean> {
|
|
174
|
+
try {
|
|
175
|
+
await stat(filePath);
|
|
176
|
+
return true;
|
|
177
|
+
} catch {
|
|
178
|
+
return false;
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
function isPathInsideDirectory(filePath: string, directory: string): boolean {
|
|
183
|
+
const relativePath = relative(directory, filePath);
|
|
184
|
+
return relativePath === '' || (!relativePath.startsWith('..') && !isAbsolute(relativePath));
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
async function resolveLocalImport(fromFile: string, specifier: string): Promise<string> {
|
|
188
|
+
const base = isAbsolute(specifier) ? resolve(specifier) : resolve(dirname(fromFile), specifier);
|
|
189
|
+
const candidates: string[] = [base];
|
|
190
|
+
const explicitExtension = extname(base).toLowerCase();
|
|
191
|
+
|
|
192
|
+
if (!explicitExtension) {
|
|
193
|
+
candidates.push(...SOURCE_EXTENSIONS.map((extension) => `${base}${extension}`));
|
|
194
|
+
candidates.push(...SOURCE_EXTENSIONS.map((extension) => join(base, `index${extension}`)));
|
|
195
|
+
} else if (['.js', '.jsx', '.mjs', '.cjs'].includes(explicitExtension)) {
|
|
196
|
+
const stem = base.slice(0, -explicitExtension.length);
|
|
197
|
+
candidates.push(...SOURCE_EXTENSIONS.map((extension) => `${stem}${extension}`));
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
for (const candidate of candidates) {
|
|
201
|
+
if (await fileExists(candidate)) {
|
|
202
|
+
return candidate;
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
throw new Error(`Could not resolve local import "${specifier}" from ${fromFile}`);
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
export async function discoverPackagedLocalFiles(
|
|
210
|
+
entryFile: string,
|
|
211
|
+
): Promise<PlayLocalFileDiscoveryResult> {
|
|
212
|
+
const absoluteEntryFile = resolve(entryFile);
|
|
213
|
+
const packagingRoot = dirname(absoluteEntryFile);
|
|
214
|
+
const files = new Map<string, PlayLocalFileReference>();
|
|
215
|
+
const unresolved: PlayLocalFileDiscoveryError[] = [];
|
|
216
|
+
const visitedFiles = new Set<string>();
|
|
217
|
+
|
|
218
|
+
const visitSourceFile = async (filePath: string): Promise<void> => {
|
|
219
|
+
const absolutePath = resolve(filePath);
|
|
220
|
+
if (visitedFiles.has(absolutePath)) {
|
|
221
|
+
return;
|
|
222
|
+
}
|
|
223
|
+
visitedFiles.add(absolutePath);
|
|
224
|
+
|
|
225
|
+
const sourceCode = await readFile(absolutePath, 'utf-8');
|
|
226
|
+
const sourceFile = ts.createSourceFile(
|
|
227
|
+
absolutePath,
|
|
228
|
+
sourceCode,
|
|
229
|
+
ts.ScriptTarget.Latest,
|
|
230
|
+
true,
|
|
231
|
+
ts.ScriptKind.TS,
|
|
232
|
+
);
|
|
233
|
+
const constants = collectTopLevelStringConstants(sourceFile);
|
|
234
|
+
const childVisits: Promise<void>[] = [];
|
|
235
|
+
|
|
236
|
+
const visitNode = async (node: ts.Node): Promise<void> => {
|
|
237
|
+
if (ts.isCallExpression(node) && isCtxCsvCall(node)) {
|
|
238
|
+
const argument = node.arguments[0];
|
|
239
|
+
if (!argument) {
|
|
240
|
+
unresolved.push({
|
|
241
|
+
sourceFragment: 'ctx.csv()',
|
|
242
|
+
message: 'ctx.csv() requires a file path string or input reference.',
|
|
243
|
+
});
|
|
244
|
+
} else if (!isRuntimeInputExpression(argument)) {
|
|
245
|
+
const resolvedPath = resolveStringExpression(argument, constants);
|
|
246
|
+
if (resolvedPath == null) {
|
|
247
|
+
unresolved.push({
|
|
248
|
+
sourceFragment: extractSourceFragment(sourceCode, argument),
|
|
249
|
+
message:
|
|
250
|
+
'Could not resolve this ctx.csv(...) path at submit time. Use a string literal, a top-level const string, or pass a runtime input like input.file.',
|
|
251
|
+
});
|
|
252
|
+
} else {
|
|
253
|
+
const absoluteCsvPath = resolve(dirname(absolutePath), resolvedPath);
|
|
254
|
+
if (isAbsolute(resolvedPath) || !isPathInsideDirectory(absoluteCsvPath, packagingRoot)) {
|
|
255
|
+
unresolved.push({
|
|
256
|
+
sourceFragment: extractSourceFragment(sourceCode, argument),
|
|
257
|
+
message:
|
|
258
|
+
'ctx.csv(...) packaged file paths must be relative paths inside the play directory. Pass external files at runtime with input.file instead.',
|
|
259
|
+
});
|
|
260
|
+
return;
|
|
261
|
+
}
|
|
262
|
+
const buffer = await readFile(absoluteCsvPath);
|
|
263
|
+
const stats = await stat(absoluteCsvPath);
|
|
264
|
+
files.set(absoluteCsvPath, {
|
|
265
|
+
sourceFragment: extractSourceFragment(sourceCode, argument),
|
|
266
|
+
logicalPath: resolvedPath,
|
|
267
|
+
absolutePath: absoluteCsvPath,
|
|
268
|
+
bytes: stats.size,
|
|
269
|
+
contentHash: sha256(buffer),
|
|
270
|
+
contentType: contentTypeForFile(absoluteCsvPath),
|
|
271
|
+
});
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
if (
|
|
277
|
+
ts.isImportDeclaration(node) &&
|
|
278
|
+
!node.importClause?.isTypeOnly &&
|
|
279
|
+
ts.isStringLiteral(node.moduleSpecifier) &&
|
|
280
|
+
node.moduleSpecifier.text.startsWith('.')
|
|
281
|
+
) {
|
|
282
|
+
childVisits.push(
|
|
283
|
+
resolveLocalImport(absolutePath, node.moduleSpecifier.text).then((resolvedImport) =>
|
|
284
|
+
visitSourceFile(resolvedImport),
|
|
285
|
+
),
|
|
286
|
+
);
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
if (
|
|
290
|
+
ts.isCallExpression(node) &&
|
|
291
|
+
ts.isIdentifier(node.expression) &&
|
|
292
|
+
node.expression.text === 'require' &&
|
|
293
|
+
node.arguments.length === 1 &&
|
|
294
|
+
ts.isStringLiteral(node.arguments[0]) &&
|
|
295
|
+
node.arguments[0].text.startsWith('.')
|
|
296
|
+
) {
|
|
297
|
+
childVisits.push(
|
|
298
|
+
resolveLocalImport(absolutePath, node.arguments[0].text).then((resolvedImport) =>
|
|
299
|
+
visitSourceFile(resolvedImport),
|
|
300
|
+
),
|
|
301
|
+
);
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
await Promise.all(node.getChildren(sourceFile).map((child) => visitNode(child)));
|
|
305
|
+
};
|
|
306
|
+
|
|
307
|
+
await visitNode(sourceFile);
|
|
308
|
+
await Promise.all(childVisits);
|
|
309
|
+
};
|
|
310
|
+
|
|
311
|
+
await visitSourceFile(absoluteEntryFile);
|
|
312
|
+
|
|
313
|
+
return {
|
|
314
|
+
files: [...files.values()],
|
|
315
|
+
unresolved,
|
|
316
|
+
};
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
export function buildPlayStorageKey(input: {
|
|
320
|
+
orgId: string;
|
|
321
|
+
contentHash: string;
|
|
322
|
+
logicalPath: string;
|
|
323
|
+
}): string {
|
|
324
|
+
const fileName = basename(input.logicalPath);
|
|
325
|
+
return `plays/v2/orgs/${input.orgId}/files/${input.contentHash}/${fileName}`;
|
|
326
|
+
}
|