@sixb/pipeline-worker 0.1.20 → 0.1.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,5 @@
1
1
  import type { DatasetDefinition, PipelineDefinition, PipelineStepDefinition } from "@sixb/core";
2
- import type { DatasetVersion } from "@sixb/core/lake-storage";
2
+ import { type DatasetVersion } from "@sixb/core/lake-storage";
3
3
  import { type ResolvedStepInput } from "./step-inputs";
4
4
  import type { PipelineJob, PipelineLogSession, PipelineWorkerContext } from "./types";
5
5
  export declare function executeRunStep(input: {
@@ -1 +1 @@
1
- {"version":3,"file":"execute-run-step.d.ts","sourceRoot":"","sources":["../src/execute-run-step.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,iBAAiB,EAEjB,kBAAkB,EAClB,sBAAsB,EAEvB,MAAM,YAAY,CAAA;AAEnB,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAA;AAE7D,OAAO,EAAoB,KAAK,iBAAiB,EAAE,MAAM,eAAe,CAAA;AACxE,OAAO,KAAK,EAAE,WAAW,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,MAAM,SAAS,CAAA;AAErF,wBAAsB,cAAc,CAAC,KAAK,EAAE;IAC1C,QAAQ,CAAC,OAAO,EAAE,qBAAqB,CAAA;IACvC,QAAQ,CAAC,QAAQ,EAAE,kBAAkB,CAAA;IACrC,QAAQ,CAAC,IAAI,EAAE,sBAAsB,CAAA;IACrC,QAAQ,CAAC,GAAG,EAAE,WAAW,CAAA;IACzB,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAA;IAC5B,QAAQ,CAAC,UAAU,EAAE,kBAAkB,CAAA;IACvC,QAAQ,CAAC,aAAa,EAAE,iBAAiB,CAAA;IACzC,QAAQ,CAAC,uBAAuB,EAAE,MAAM,GAAG,IAAI,CAAA;IAC/C,QAAQ,CAAC,cAAc,EAAE,SAAS,iBAAiB,EAAE,CAAA;CACtD,GAAG,OAAO,CAAC;IACV,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAA;IAChC,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAA;IAChC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;CAC7B,CAAC,CAsED"}
1
+ {"version":3,"file":"execute-run-step.d.ts","sourceRoot":"","sources":["../src/execute-run-step.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,iBAAiB,EAEjB,kBAAkB,EAClB,sBAAsB,EAEvB,MAAM,YAAY,CAAA;AAGnB,OAAO,EAAE,KAAK,cAAc,EAAsB,MAAM,yBAAyB,CAAA;AAEjF,OAAO,EAAoB,KAAK,iBAAiB,EAAE,MAAM,eAAe,CAAA;AACxE,OAAO,KAAK,EAAE,WAAW,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,MAAM,SAAS,CAAA;AAErF,wBAAsB,cAAc,CAAC,KAAK,EAAE;IAC1C,QAAQ,CAAC,OAAO,EAAE,qBAAqB,CAAA;IACvC,QAAQ,CAAC,QAAQ,EAAE,kBAAkB,CAAA;IACrC,QAAQ,CAAC,IAAI,EAAE,sBAAsB,CAAA;IACrC,QAAQ,CAAC,GAAG,EAAE,WAAW,CAAA;IACzB,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAA;IAC5B,QAAQ,CAAC,UAAU,EAAE,kBAAkB,CAAA;IACvC,QAAQ,CAAC,aAAa,EAAE,iBAAiB,CAAA;IACzC,QAAQ,CAAC,uBAAuB,EAAE,MAAM,GAAG,IAAI,CAAA;IAC/C,QAAQ,CAAC,cAAc,EAAE,SAAS,iBAAiB,EAAE,CAAA;CACtD,GAAG,OAAO,CAAC;IACV,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAA;IAChC,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAA;IAChC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;CAC7B,CAAC,CA6ED"}
@@ -1 +1 @@
1
- {"version":3,"file":"execute-sql-step.d.ts","sourceRoot":"","sources":["../src/execute-sql-step.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,iBAAiB,EAGjB,kBAAkB,EAClB,sBAAsB,EACvB,MAAM,YAAY,CAAA;AAEnB,OAAO,KAAK,EACV,cAAc,EAGf,MAAM,yBAAyB,CAAA;AAEhC,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAA;AACtD,OAAO,KAAK,EAAE,WAAW,EAAE,qBAAqB,EAAE,MAAM,SAAS,CAAA;AAwCjE,wBAAsB,cAAc,CAAC,KAAK,EAAE;IAC1C,QAAQ,CAAC,OAAO,EAAE,qBAAqB,CAAA;IACvC,QAAQ,CAAC,QAAQ,EAAE,kBAAkB,CAAA;IACrC,QAAQ,CAAC,IAAI,EAAE,sBAAsB,CAAA;IACrC,QAAQ,CAAC,GAAG,EAAE,WAAW,CAAA;IACzB,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAA;IAC5B,QAAQ,CAAC,aAAa,EAAE,iBAAiB,CAAA;IACzC,QAAQ,CAAC,uBAAuB,EAAE,MAAM,GAAG,IAAI,CAAA;IAC/C,QAAQ,CAAC,cAAc,EAAE,SAAS,iBAAiB,EAAE,CAAA;CACtD,GAAG,OAAO,CAAC;IACV,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAA;IAChC,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAA;IAChC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAC9B,CAAC,CAgFD"}
1
+ {"version":3,"file":"execute-sql-step.d.ts","sourceRoot":"","sources":["../src/execute-sql-step.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,iBAAiB,EAGjB,kBAAkB,EAClB,sBAAsB,EACvB,MAAM,YAAY,CAAA;AAEnB,OAAO,KAAK,EACV,cAAc,EAGf,MAAM,yBAAyB,CAAA;AAEhC,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAA;AACtD,OAAO,KAAK,EAAE,WAAW,EAAE,qBAAqB,EAAE,MAAM,SAAS,CAAA;AAwCjE,wBAAsB,cAAc,CAAC,KAAK,EAAE;IAC1C,QAAQ,CAAC,OAAO,EAAE,qBAAqB,CAAA;IACvC,QAAQ,CAAC,QAAQ,EAAE,kBAAkB,CAAA;IACrC,QAAQ,CAAC,IAAI,EAAE,sBAAsB,CAAA;IACrC,QAAQ,CAAC,GAAG,EAAE,WAAW,CAAA;IACzB,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAA;IAC5B,QAAQ,CAAC,aAAa,EAAE,iBAAiB,CAAA;IACzC,QAAQ,CAAC,uBAAuB,EAAE,MAAM,GAAG,IAAI,CAAA;IAC/C,QAAQ,CAAC,cAAc,EAAE,SAAS,iBAAiB,EAAE,CAAA;CACtD,GAAG,OAAO,CAAC;IACV,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAA;IAChC,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAA;IAChC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAC9B,CAAC,CAiFD"}
package/dist/index.js CHANGED
@@ -218,9 +218,12 @@ import { PIPELINE_RUN_FAILURE_CODES } from "@sixb/core/storage";
218
218
 
219
219
  // src/execute-run-step.ts
220
220
  import { createSixbError as createSixbError3 } from "@sixb/core/internal/errors";
221
+ import { runStepLineage } from "@sixb/core/internal/pipelines";
222
+ import { resolveStepLineage } from "@sixb/core/lake-storage";
221
223
 
222
224
  // src/step-inputs.ts
223
225
  import { createSixbError as createSixbError2 } from "@sixb/core/internal/errors";
226
+ import { isDatasetPick, stepInputColumns } from "@sixb/core/internal/pipelines";
224
227
  async function resolveStepInputs(input) {
225
228
  const { runtime, pipeline, step, pipelineRunId } = input;
226
229
  const resolved = [];
@@ -248,6 +251,7 @@ async function resolveStepInputs(input) {
248
251
  resolved.push({
249
252
  name,
250
253
  dataset,
254
+ ...isDatasetPick(declaredDataset) ? { pick: { dataset: declaredDataset, columns: stepInputColumns(declaredDataset) } } : {},
251
255
  version,
252
256
  ref: {
253
257
  datasetId: version.datasetId,
@@ -257,24 +261,37 @@ async function resolveStepInputs(input) {
257
261
  }
258
262
  return resolved;
259
263
  }
260
- function createStepInputs(lakeStorage, resolved, signal) {
264
+ function createStepInputs(input) {
265
+ const { lakeStorage, signal } = input;
261
266
  const inputs = {};
262
- for (const input of resolved) {
263
- inputs[input.name] = {
264
- dataset: input.dataset,
265
- version: input.version,
267
+ for (const resolved of input.resolvedInputs) {
268
+ const subject = `[SixbPipelineWorker] Pipeline '${input.pipeline.id}' step '${input.step.id}' input '${resolved.name}'`;
269
+ const { pick } = resolved;
270
+ inputs[resolved.name] = {
271
+ dataset: pick?.dataset ?? resolved.dataset,
272
+ version: resolved.version,
266
273
  readRows(readInput = {}) {
267
274
  return lakeStorage.readRows({
268
275
  ...readInput,
276
+ ...pick && { columns: readPickedColumns(subject, pick.columns, readInput.columns) },
269
277
  signal: readInput.signal ? AbortSignal.any([signal, readInput.signal]) : signal,
270
- datasetId: input.dataset.id,
271
- versionId: input.version.versionId
278
+ datasetId: resolved.dataset.id,
279
+ versionId: resolved.version.versionId
272
280
  });
273
281
  }
274
282
  };
275
283
  }
276
284
  return inputs;
277
285
  }
286
+ function readPickedColumns(subject, pickedColumns, requested) {
287
+ if (requested === undefined || requested.length === 0)
288
+ return pickedColumns;
289
+ const unpicked = requested.find((column) => !pickedColumns.includes(column));
290
+ if (unpicked !== undefined) {
291
+ throw new Error(`${subject} reads column '${unpicked}', which its pick leaves out. Add '${unpicked}' to the pick to read it.`);
292
+ }
293
+ return requested;
294
+ }
278
295
 
279
296
  // src/execute-run-step.ts
280
297
  async function executeRunStep(input) {
@@ -295,12 +312,8 @@ async function executeRunStep(input) {
295
312
  dataset: outputDataset,
296
313
  mode: step.mode,
297
314
  inputs: resolvedInputs.map((resolved) => resolved.ref),
298
- producer: {
299
- kind: "pipeline",
300
- id: pipeline.id,
301
- runId: job.id,
302
- stepId: step.id
303
- }
315
+ producer: { kind: "pipeline", id: pipeline.id, runId: job.id, stepId: step.id },
316
+ lineage: resolveStepLineage(runStepLineage(step, new Map([[outputDataset.id, outputDataset]])), resolvedInputs.map((resolved) => resolved.version))
304
317
  });
305
318
  try {
306
319
  const context = {
@@ -310,7 +323,13 @@ async function executeRunStep(input) {
310
323
  runId: job.id,
311
324
  signal,
312
325
  logger,
313
- inputs: createStepInputs(runtime.lakeStorage, resolvedInputs, signal),
326
+ inputs: createStepInputs({
327
+ lakeStorage: runtime.lakeStorage,
328
+ pipeline,
329
+ step,
330
+ resolvedInputs,
331
+ signal
332
+ }),
314
333
  output: {
315
334
  async writeRows(rows) {
316
335
  await write.writeRows(countRowsWritten(rows, signal, () => {
@@ -400,7 +419,8 @@ async function executeSqlStep(input) {
400
419
  input.name,
401
420
  {
402
421
  dataset: input.dataset,
403
- versionId: input.version.versionId
422
+ versionId: input.version.versionId,
423
+ ...input.pick && { columns: input.pick.columns }
404
424
  }
405
425
  ])),
406
426
  sql: step.executor.sql,
@@ -813,4 +833,4 @@ export {
813
833
  PipelineWorker
814
834
  };
815
835
 
816
- //# debugId=71A662855CE2F2BB64756E2164756E21
836
+ //# debugId=5FFC283C2E39D35B64756E2164756E21
package/dist/index.js.map CHANGED
@@ -7,11 +7,11 @@
7
7
  "import { captureSixbFailure } from \"@sixb/core/internal/errors\"\nimport { resolveLoggingService } from \"@sixb/core/internal/logging\"\nimport type { DatasetVersion } from \"@sixb/core/lake-storage\"\nimport { PIPELINE_RUN_FAILURE_CODES, type PipelineRunRecord } from \"@sixb/core/storage\"\nimport {\n createPipelineBookkeepingError,\n requireFinishedAt,\n requirePipeline,\n statusForFailure,\n throwIfAborted,\n unwrapPipelineStepFailure,\n} from \"./errors\"\nimport { runStep } from \"./run-step\"\nimport type {\n PipelineRunFinishedHandler,\n PipelineRunResult,\n PipelineStepRunResult,\n RunPipelineJobInput,\n} from \"./types\"\n\nexport class PipelineRunAlreadyStartedError extends Error {\n override readonly name = \"PipelineRunAlreadyStartedError\"\n\n constructor(readonly run: PipelineRunRecord) {\n super(`[SixbPipelineWorker] Pipeline run '${run.id}' has already started.`)\n }\n}\n\nexport async function runPipelineJob(input: RunPipelineJobInput): Promise<PipelineRunResult> {\n const { runtime, run: durableRun } = input\n const job = { id: durableRun.id, pipelineId: durableRun.pipelineId }\n const signal = input.signal ?? new AbortController().signal\n if (durableRun.projectId !== runtime.id) {\n throw new Error(\n `[SixbPipelineWorker] Pipeline run '${durableRun.id}' belongs to project '${durableRun.projectId}', not '${runtime.id}'.`\n )\n }\n const logSession = resolveLoggingService(runtime.id, runtime.logging).startExecution({\n kind: \"pipeline\",\n id: job.id,\n })\n const steps: PipelineStepRunResult[] = []\n let startedRun: PipelineRunRecord | undefined\n let finalVersion: DatasetVersion | undefined\n let finished = false\n\n throwIfAborted(signal)\n\n try {\n try {\n startedRun = await runtime.pipelineRunsStorage.start({\n projectId: runtime.id,\n id: job.id,\n })\n } catch (error) {\n const existing = await runtime.pipelineRunsStorage.getById({\n projectId: runtime.id,\n id: job.id,\n })\n if (\n existing?.pipelineId === durableRun.pipelineId &&\n existing.executionId === durableRun.executionId\n ) {\n throw new PipelineRunAlreadyStartedError(existing)\n }\n throw error\n }\n await input.onRunStarted?.(startedRun)\n\n const pipeline = requirePipeline(runtime.pipelines.getById(job.pipelineId), job)\n\n for (const [stepIndex, node] of pipeline.graph.nodes.entries()) {\n const stepResult = await runStep({\n runtime,\n pipeline,\n step: node.step,\n stepIndex,\n job,\n signal,\n logSession,\n onStepStarted: input.onStepStarted,\n onStepFinished: input.onStepFinished,\n })\n\n steps.push(stepResult)\n finalVersion = stepResult.version\n // This is the durability boundary for per-step output events.\n await input.onStepCommitted?.(stepResult)\n }\n\n let run: PipelineRunRecord\n try {\n run = await runtime.pipelineRunsStorage.finish({\n projectId: runtime.id,\n id: job.id,\n status: \"succeeded\",\n output: finalVersion\n ? {\n datasetId: finalVersion.datasetId,\n versionId: finalVersion.versionId,\n }\n : undefined,\n })\n } catch (error) {\n if (finalVersion) {\n throw createPipelineBookkeepingError({\n pipelineId: pipeline.id,\n runId: job.id,\n version: finalVersion,\n cause: error,\n })\n }\n throw error\n }\n const finishedRun = {\n ...run,\n finishedAt: requireFinishedAt({\n pipelineId: pipeline.id,\n runId: job.id,\n finishedAt: run.finishedAt,\n }),\n }\n\n finished = true\n await notifyRunFinished(input.onRunFinished, finishedRun)\n return {\n run: finishedRun,\n steps,\n version: finalVersion,\n }\n } catch (error) {\n const reportedError = unwrapPipelineStepFailure(error)\n if (startedRun && !finished) {\n const status = statusForFailure(signal, error)\n try {\n const failure = captureSixbFailure(error, {\n allowedCodes: PIPELINE_RUN_FAILURE_CODES,\n defaultCode: status === \"cancelled\" ? \"runtime.cancelled\" : \"internal.unexpected\",\n details: { pipelineId: durableRun.pipelineId, runId: job.id },\n })\n const run = await runtime.pipelineRunsStorage.finish({\n projectId: runtime.id,\n id: job.id,\n status,\n error: failure,\n })\n if (status === \"failed\" && run.status === \"failed\") {\n input.onRunFailed?.(reportedError, run, failure)\n }\n await notifyRunFinished(input.onRunFinished, run)\n } catch {\n // The run did not transition to the requested terminal status.\n }\n }\n\n throw reportedError\n } finally {\n await logSession.flush()\n }\n}\n\nasync function notifyRunFinished(\n handler: PipelineRunFinishedHandler | undefined,\n run: PipelineRunRecord\n): Promise<void> {\n try {\n await handler?.(run)\n } catch (error) {\n // The built-in event log reports lost batches itself. Anything reaching here is a broken\n // invariant in a custom lifecycle handler and must not change an already durable outcome.\n console.error(\"[SixbPipelineWorker] Pipeline run lifecycle handler failed:\", error)\n }\n}\n",
8
8
  "import type { DatasetDefinition, PipelineDefinition } from \"@sixb/core\"\nimport { createSixbError, isSixbError, summarizeErrorMessage } from \"@sixb/core/internal/errors\"\nimport type { DatasetVersion } from \"@sixb/core/lake-storage\"\nimport type { PipelineRunStatus } from \"@sixb/core/storage\"\nimport type { PipelineJob } from \"./types\"\n\nexport function throwIfAborted(signal: AbortSignal): void {\n if (signal.aborted) {\n throw createAbortError()\n }\n}\n\nexport function statusForFailure(\n signal: AbortSignal,\n error: unknown\n): Extract<PipelineRunStatus, \"failed\" | \"cancelled\"> {\n return signal.aborted || isAbortError(error) ? \"cancelled\" : \"failed\"\n}\n\nexport function requireFinishedAt(input: {\n readonly pipelineId: string\n readonly runId: string\n readonly finishedAt: Date | undefined\n}): Date {\n if (input.finishedAt) {\n return input.finishedAt\n }\n\n throw createSixbError(\n \"internal.unexpected\",\n `[SixbPipelineWorker] Pipeline run '${input.runId}' finished without a finishedAt timestamp.`,\n { details: { pipelineId: input.pipelineId, runId: input.runId } }\n )\n}\n\nexport function createStepBookkeepingError(options: {\n readonly pipelineId: string\n readonly pipelineRunId: string\n readonly stepId: string\n readonly stepRunId: string\n readonly version: DatasetVersion\n readonly cause: unknown\n}): Error {\n // The dataset version is already durable; retrying the whole pipeline could duplicate appends.\n return createSixbError(\n \"internal.unexpected\",\n `[SixbPipelineWorker] Pipeline '${options.pipelineId}' step '${options.stepId}' committed dataset version '${options.version.versionId}', but failed to finalize step run '${options.stepRunId}'. The dataset commit may already have succeeded and the step run record may need repair.`,\n {\n cause: options.cause,\n details: {\n pipelineId: options.pipelineId,\n pipelineRunId: options.pipelineRunId,\n stepId: options.stepId,\n stepRunId: options.stepRunId,\n datasetId: options.version.datasetId,\n versionId: options.version.versionId,\n },\n }\n )\n}\n\nexport function createPipelineStepFailure(options: {\n readonly pipelineId: string\n readonly pipelineRunId: string\n readonly stepId: string\n readonly stepRunId?: string\n readonly cause: unknown\n}) {\n return createSixbError(\n \"pipeline.step_failed\",\n summarizeErrorMessage(options.cause, \"Pipeline step execution failed.\"),\n {\n cause: options.cause,\n details: {\n pipelineId: options.pipelineId,\n pipelineRunId: options.pipelineRunId,\n stepId: options.stepId,\n ...(options.stepRunId ? { stepRunId: options.stepRunId } : {}),\n },\n }\n )\n}\n\nexport function unwrapPipelineStepFailure(error: unknown): unknown {\n return isSixbError(error) && error.code === \"pipeline.step_failed\" && error.cause !== undefined\n ? error.cause\n : error\n}\n\nexport function createPipelineBookkeepingError(options: {\n readonly pipelineId: string\n readonly runId: string\n readonly version: DatasetVersion\n readonly cause: unknown\n}): Error {\n return createSixbError(\n \"internal.unexpected\",\n `[SixbPipelineWorker] Pipeline '${options.pipelineId}' committed final dataset version '${options.version.versionId}', but failed to finalize pipeline run '${options.runId}'. The dataset commit may already have succeeded and the pipeline run record may need repair.`,\n {\n cause: options.cause,\n details: {\n pipelineId: options.pipelineId,\n runId: options.runId,\n datasetId: options.version.datasetId,\n versionId: options.version.versionId,\n },\n }\n )\n}\n\nexport function requirePipeline(\n pipeline: PipelineDefinition | null,\n job: PipelineJob\n): PipelineDefinition {\n if (!pipeline) {\n throw createSixbError(\n \"internal.unexpected\",\n `[SixbPipelineWorker] Unknown pipeline '${job.pipelineId}'.`,\n { details: { pipelineId: job.pipelineId, runId: job.id } }\n )\n }\n\n return pipeline\n}\n\nexport function requireRegisteredDataset(options: {\n readonly dataset: DatasetDefinition | null\n readonly pipelineId: string\n readonly pipelineRunId: string\n readonly stepId: string\n readonly role: \"input\" | \"output\"\n readonly name?: string\n readonly datasetId: string\n}): DatasetDefinition {\n if (options.dataset) {\n return options.dataset\n }\n\n const namedRole = options.role === \"input\" ? `input '${options.name ?? \"unknown\"}'` : \"output\"\n\n throw createSixbError(\n \"internal.unexpected\",\n `[SixbPipelineWorker] Pipeline '${options.pipelineId}' step '${options.stepId}' ${namedRole} references unknown dataset '${options.datasetId}'.`,\n {\n details: {\n pipelineId: options.pipelineId,\n pipelineRunId: options.pipelineRunId,\n stepId: options.stepId,\n datasetId: options.datasetId,\n },\n }\n )\n}\n\nfunction createAbortError(): Error {\n const error = new Error(\"Worker runtime aborted.\")\n error.name = \"AbortError\"\n return error\n}\n\nfunction isAbortError(error: unknown): boolean {\n return error instanceof Error && error.name === \"AbortError\"\n}\n",
9
9
  "import type { DatasetDefinition, PipelineDefinition, PipelineStepDefinition } from \"@sixb/core\"\nimport { lakeDatasetDefinition } from \"@sixb/core/internal/datasets\"\nimport { captureSixbFailure } from \"@sixb/core/internal/errors\"\nimport type { DatasetVersion } from \"@sixb/core/lake-storage\"\nimport { PIPELINE_RUN_FAILURE_CODES, type PipelineStepRunRecord } from \"@sixb/core/storage\"\nimport {\n createPipelineStepFailure,\n createStepBookkeepingError,\n requireRegisteredDataset,\n statusForFailure,\n throwIfAborted,\n} from \"./errors\"\nimport { executeRunStep } from \"./execute-run-step\"\nimport { executeSqlStep } from \"./execute-sql-step\"\nimport { type ResolvedStepInput, resolveStepInputs } from \"./step-inputs\"\nimport type {\n PipelineJob,\n PipelineLogSession,\n PipelineStepFinishedHandler,\n PipelineStepRunResult,\n PipelineStepStartedHandler,\n PipelineWorkerContext,\n} from \"./types\"\n\ntype StepExecutionResult = {\n readonly version: DatasetVersion\n readonly versionCreated: boolean\n readonly rowsWritten?: number\n}\n\nexport async function runStep(input: {\n readonly runtime: PipelineWorkerContext\n readonly pipeline: PipelineDefinition\n readonly step: PipelineStepDefinition\n readonly stepIndex: number\n readonly job: PipelineJob\n readonly signal: AbortSignal\n readonly logSession: PipelineLogSession\n readonly onStepStarted?: PipelineStepStartedHandler\n readonly onStepFinished?: PipelineStepFinishedHandler\n}): Promise<PipelineStepRunResult> {\n const { runtime, pipeline, step, stepIndex, job, signal } = input\n const lifecycleContext = {\n stepIndex,\n totalSteps: pipeline.graph.nodes.length,\n }\n\n throwIfAborted(signal)\n let resolvedInputs: readonly ResolvedStepInput[]\n let outputDataset: DatasetDefinition\n let expectedLatestVersionId: string | null\n try {\n outputDataset = lakeDatasetDefinition(\n requireRegisteredDataset({\n dataset: runtime.datasets.getById(step.output.id),\n pipelineId: pipeline.id,\n pipelineRunId: job.id,\n stepId: step.id,\n role: \"output\",\n datasetId: step.output.id,\n })\n )\n\n await runtime.lakeStorage.createDataset(outputDataset)\n // Capture the output guard before any input is pinned. A later capture could legitimize\n // overwriting a concurrent run with results computed from older inputs.\n expectedLatestVersionId =\n (await runtime.lakeStorage.getLatestVersion(outputDataset.id))?.versionId ?? null\n resolvedInputs = await resolveStepInputs({\n runtime,\n pipeline,\n step,\n pipelineRunId: job.id,\n })\n } catch (error) {\n if (statusForFailure(signal, error) === \"failed\") {\n throw createPipelineStepFailure({\n pipelineId: pipeline.id,\n pipelineRunId: job.id,\n stepId: step.id,\n cause: error,\n })\n }\n throw error\n }\n const inputRefs = resolvedInputs.map((resolved) => resolved.ref)\n\n const stepRunId = `${job.id}:step:${stepIndex + 1}:${step.id}`\n const startedStepRun = await runtime.pipelineRunsStorage.startStep({\n projectId: runtime.id,\n id: stepRunId,\n pipelineRunId: job.id,\n pipelineId: pipeline.id,\n stepId: step.id,\n datasetId: outputDataset.id,\n mode: step.mode,\n inputs: inputRefs,\n })\n await input.onStepStarted?.(startedStepRun, lifecycleContext)\n\n let committedVersion: DatasetVersion | undefined\n let rowsWritten: number | undefined\n\n try {\n const execution = await executeStep({\n runtime,\n pipeline,\n step,\n job,\n signal,\n logSession: input.logSession,\n outputDataset,\n expectedLatestVersionId,\n resolvedInputs,\n })\n committedVersion = execution.version\n rowsWritten = execution.rowsWritten\n\n let stepRun: PipelineStepRunRecord\n try {\n stepRun = await runtime.pipelineRunsStorage.finishStep({\n projectId: runtime.id,\n id: stepRunId,\n status: \"succeeded\",\n output: {\n datasetId: committedVersion.datasetId,\n versionId: committedVersion.versionId,\n },\n rowsWritten,\n })\n } catch (error) {\n throw createStepBookkeepingError({\n pipelineId: pipeline.id,\n pipelineRunId: job.id,\n stepId: step.id,\n stepRunId,\n version: committedVersion,\n cause: error,\n })\n }\n await input.onStepFinished?.(stepRun, lifecycleContext)\n\n return {\n run: stepRun,\n version: committedVersion,\n versionCreated: execution.versionCreated,\n }\n } catch (error) {\n if (!committedVersion) {\n const status = statusForFailure(signal, error)\n const failureError =\n status === \"failed\"\n ? createPipelineStepFailure({\n pipelineId: pipeline.id,\n pipelineRunId: job.id,\n stepId: step.id,\n stepRunId,\n cause: error,\n })\n : error\n const failedStepRun = await runtime.pipelineRunsStorage\n .finishStep({\n projectId: runtime.id,\n id: stepRunId,\n status,\n rowsWritten,\n error: captureSixbFailure(failureError, {\n allowedCodes: PIPELINE_RUN_FAILURE_CODES,\n defaultCode: status === \"cancelled\" ? \"runtime.cancelled\" : \"internal.unexpected\",\n details: {\n pipelineId: pipeline.id,\n pipelineRunId: job.id,\n stepId: step.id,\n stepRunId,\n },\n }),\n })\n .catch(() => null)\n\n if (failedStepRun) {\n await input.onStepFinished?.(failedStepRun, lifecycleContext)\n }\n\n throw failureError\n }\n\n throw error\n }\n}\n\nasync function executeStep(input: {\n readonly runtime: PipelineWorkerContext\n readonly pipeline: PipelineDefinition\n readonly step: PipelineStepDefinition\n readonly job: PipelineJob\n readonly signal: AbortSignal\n readonly logSession: PipelineLogSession\n readonly outputDataset: DatasetDefinition\n readonly expectedLatestVersionId: string | null\n readonly resolvedInputs: readonly ResolvedStepInput[]\n}): Promise<StepExecutionResult> {\n switch (input.step.executor.kind) {\n case \"run\":\n return executeRunStep(input)\n case \"sql\":\n return executeSqlStep(input)\n }\n}\n",
10
- "import type {\n DatasetDefinition,\n DatasetRow,\n PipelineDefinition,\n PipelineStepDefinition,\n PipelineStepRunContext,\n} from \"@sixb/core\"\nimport { createSixbError } from \"@sixb/core/internal/errors\"\nimport type { DatasetVersion } from \"@sixb/core/lake-storage\"\nimport { throwIfAborted } from \"./errors\"\nimport { createStepInputs, type ResolvedStepInput } from \"./step-inputs\"\nimport type { PipelineJob, PipelineLogSession, PipelineWorkerContext } from \"./types\"\n\nexport async function executeRunStep(input: {\n readonly runtime: PipelineWorkerContext\n readonly pipeline: PipelineDefinition\n readonly step: PipelineStepDefinition\n readonly job: PipelineJob\n readonly signal: AbortSignal\n readonly logSession: PipelineLogSession\n readonly outputDataset: DatasetDefinition\n readonly expectedLatestVersionId: string | null\n readonly resolvedInputs: readonly ResolvedStepInput[]\n}): Promise<{\n readonly version: DatasetVersion\n readonly versionCreated: boolean\n readonly rowsWritten: number\n}> {\n const { runtime, pipeline, step, job, signal, outputDataset, resolvedInputs } = input\n\n if (step.executor.kind !== \"run\") {\n throw createSixbError(\n \"internal.unexpected\",\n `[SixbPipelineWorker] Pipeline '${pipeline.id}' step '${step.id}' uses SQL execution, which is not supported by the run step executor.`,\n {\n details: {\n pipelineId: pipeline.id,\n pipelineRunId: job.id,\n stepId: step.id,\n datasetId: outputDataset.id,\n },\n }\n )\n }\n\n const logger = input.logSession.withContext({ stepId: step.id })\n let rowsWritten = 0\n const write = await runtime.lakeStorage.beginWrite({\n dataset: outputDataset,\n mode: step.mode,\n inputs: resolvedInputs.map((resolved) => resolved.ref),\n producer: {\n kind: \"pipeline\",\n id: pipeline.id,\n runId: job.id,\n stepId: step.id,\n },\n })\n\n try {\n const context: PipelineStepRunContext = {\n projectId: runtime.id,\n pipelineId: pipeline.id,\n stepId: step.id,\n runId: job.id,\n signal,\n logger,\n inputs: createStepInputs(runtime.lakeStorage, resolvedInputs, signal),\n output: {\n async writeRows(rows) {\n await write.writeRows(\n countRowsWritten(rows, signal, () => {\n rowsWritten += 1\n })\n )\n },\n },\n }\n\n await step.executor.handler(context)\n throwIfAborted(signal)\n\n const commit = await write.commit({\n expectedLatestVersionId: input.expectedLatestVersionId,\n commitMessage: `pipeline ${pipeline.id} step ${step.id} run ${job.id}`,\n })\n const { outcome, ...version } = commit\n\n return {\n version,\n versionCreated: outcome === \"created\",\n rowsWritten,\n }\n } catch (error) {\n await write.abort().catch(() => {})\n throw error\n }\n}\n\nasync function* countRowsWritten(\n rows: Iterable<DatasetRow> | AsyncIterable<DatasetRow>,\n signal: AbortSignal,\n onWritten: () => void\n): AsyncIterable<DatasetRow> {\n for await (const row of rows) {\n throwIfAborted(signal)\n yield row\n onWritten()\n }\n}\n",
11
- "import type {\n DatasetDefinition,\n PipelineDefinition,\n PipelineStepDefinition,\n PipelineStepInput,\n PipelineStepRunContext,\n} from \"@sixb/core\"\nimport { createSixbError } from \"@sixb/core/internal/errors\"\nimport type {\n DatasetVersion,\n DatasetVersionRef,\n ReadDatasetRowsInput,\n} from \"@sixb/core/lake-storage\"\nimport { requireRegisteredDataset } from \"./errors\"\nimport type { PipelineWorkerContext } from \"./types\"\n\nexport interface ResolvedStepInput {\n readonly name: string\n readonly dataset: DatasetDefinition\n readonly version: DatasetVersion\n readonly ref: DatasetVersionRef\n}\n\nexport async function resolveStepInputs(input: {\n readonly runtime: PipelineWorkerContext\n readonly pipeline: PipelineDefinition\n readonly step: PipelineStepDefinition\n readonly pipelineRunId: string\n}): Promise<readonly ResolvedStepInput[]> {\n const { runtime, pipeline, step, pipelineRunId } = input\n const resolved: ResolvedStepInput[] = []\n\n for (const [name, declaredDataset] of Object.entries(step.inputs)) {\n const dataset = requireRegisteredDataset({\n dataset: runtime.datasets.getById(declaredDataset.id),\n pipelineId: pipeline.id,\n pipelineRunId,\n stepId: step.id,\n role: \"input\",\n name,\n datasetId: declaredDataset.id,\n })\n\n const version = await runtime.lakeStorage.getLatestVersion(dataset.id)\n if (!version) {\n throw createSixbError(\n \"internal.unexpected\",\n `[SixbPipelineWorker] Pipeline '${pipeline.id}' step '${step.id}' input '${name}' dataset '${dataset.id}' has no committed version.`,\n {\n details: {\n pipelineId: pipeline.id,\n pipelineRunId,\n stepId: step.id,\n datasetId: dataset.id,\n },\n }\n )\n }\n\n resolved.push({\n name,\n dataset,\n version,\n ref: {\n datasetId: version.datasetId,\n versionId: version.versionId,\n },\n })\n }\n\n return resolved\n}\n\nexport function createStepInputs(\n lakeStorage: PipelineWorkerContext[\"lakeStorage\"],\n resolved: readonly ResolvedStepInput[],\n signal: AbortSignal\n): PipelineStepRunContext[\"inputs\"] {\n const inputs: Record<string, PipelineStepInput> = {}\n\n for (const input of resolved) {\n inputs[input.name] = {\n dataset: input.dataset,\n version: input.version,\n readRows(readInput: Omit<ReadDatasetRowsInput, \"datasetId\" | \"versionId\"> = {}) {\n // Step readers stay pinned even if newer versions commit while the handler runs.\n return lakeStorage.readRows({\n ...readInput,\n signal: readInput.signal ? AbortSignal.any([signal, readInput.signal]) : signal,\n datasetId: input.dataset.id,\n versionId: input.version.versionId,\n })\n },\n }\n }\n\n return inputs\n}\n",
12
- "import type {\n DatasetDefinition,\n LakeStorage,\n LakeStorageWithSql,\n PipelineDefinition,\n PipelineStepDefinition,\n} from \"@sixb/core\"\nimport { createSixbError } from \"@sixb/core/internal/errors\"\nimport type {\n DatasetVersion,\n DatasetWriteMode,\n LakeSqlTransformCapabilities,\n} from \"@sixb/core/lake-storage\"\nimport { throwIfAborted } from \"./errors\"\nimport type { ResolvedStepInput } from \"./step-inputs\"\nimport type { PipelineJob, PipelineWorkerContext } from \"./types\"\n\nfunction hasSqlExecutor(lakeStorage: LakeStorage): lakeStorage is LakeStorageWithSql {\n const sql = \"sql\" in lakeStorage ? lakeStorage.sql : null\n\n return (\n typeof sql === \"object\" && sql !== null && \"execute\" in sql && typeof sql.execute === \"function\"\n )\n}\n\nfunction assertSqlModeSupported(input: {\n readonly capabilities: LakeSqlTransformCapabilities\n readonly dialect: string\n readonly mode: DatasetWriteMode\n readonly pipelineId: string\n readonly pipelineRunId: string\n readonly stepId: string\n readonly datasetId: string\n}): void {\n const supported =\n input.mode === \"append\"\n ? input.capabilities.supportsAppend\n : input.capabilities.supportsSnapshot\n if (supported) return\n\n throw createSixbError(\n \"internal.unexpected\",\n `[SixbPipelineWorker] Pipeline '${input.pipelineId}' step '${input.stepId}' writes in ` +\n `'${input.mode}' mode, which the ${input.dialect} SQL executor does not support.`,\n {\n details: {\n pipelineId: input.pipelineId,\n pipelineRunId: input.pipelineRunId,\n stepId: input.stepId,\n datasetId: input.datasetId,\n },\n }\n )\n}\n\nexport async function executeSqlStep(input: {\n readonly runtime: PipelineWorkerContext\n readonly pipeline: PipelineDefinition\n readonly step: PipelineStepDefinition\n readonly job: PipelineJob\n readonly signal: AbortSignal\n readonly outputDataset: DatasetDefinition\n readonly expectedLatestVersionId: string | null\n readonly resolvedInputs: readonly ResolvedStepInput[]\n}): Promise<{\n readonly version: DatasetVersion\n readonly versionCreated: boolean\n readonly rowsWritten?: number\n}> {\n const { runtime, pipeline, step, job, signal, outputDataset, resolvedInputs } = input\n\n if (step.executor.kind !== \"sql\") {\n throw createSixbError(\n \"internal.unexpected\",\n `[SixbPipelineWorker] Pipeline '${pipeline.id}' step '${step.id}' uses run execution, which is not supported by the SQL step executor.`,\n {\n details: {\n pipelineId: pipeline.id,\n pipelineRunId: job.id,\n stepId: step.id,\n datasetId: outputDataset.id,\n },\n }\n )\n }\n\n if (!hasSqlExecutor(runtime.lakeStorage)) {\n throw createSixbError(\n \"internal.unexpected\",\n `[SixbPipelineWorker] Pipeline '${pipeline.id}' step '${step.id}' requires SQL transform support, but lake storage does not provide lakeStorage.sql.execute(...).`,\n {\n details: {\n pipelineId: pipeline.id,\n pipelineRunId: job.id,\n stepId: step.id,\n datasetId: outputDataset.id,\n },\n }\n )\n }\n\n // `LakeSqlExecutor.capabilities` is a required field on the contract that nothing read,\n // so a provider declaring `supportsAppend: false` was asked to be honest and then\n // ignored — the unsupported mode failed somewhere inside the provider instead, in\n // whatever words that provider happened to use.\n assertSqlModeSupported({\n capabilities: runtime.lakeStorage.sql.capabilities,\n dialect: runtime.lakeStorage.sql.dialect,\n mode: step.mode,\n pipelineId: pipeline.id,\n pipelineRunId: job.id,\n stepId: step.id,\n datasetId: outputDataset.id,\n })\n\n throwIfAborted(signal)\n\n const commit = await runtime.lakeStorage.sql.execute({\n sources: Object.fromEntries(\n resolvedInputs.map((input) => [\n input.name,\n {\n dataset: input.dataset,\n versionId: input.version.versionId,\n },\n ])\n ),\n sql: step.executor.sql,\n target: outputDataset,\n expectedLatestVersionId: input.expectedLatestVersionId,\n mode: step.mode,\n producer: {\n kind: \"pipeline\",\n id: pipeline.id,\n runId: job.id,\n stepId: step.id,\n },\n commitMessage: `pipeline ${pipeline.id} step ${step.id} run ${job.id}`,\n })\n\n throwIfAborted(signal)\n const { outcome, ...version } = commit\n\n return {\n version,\n versionCreated: outcome === \"created\",\n rowsWritten: version.rowCount,\n }\n}\n"
10
+ "import type {\n DatasetDefinition,\n DatasetRow,\n PipelineDefinition,\n PipelineStepDefinition,\n PipelineStepRunContext,\n} from \"@sixb/core\"\nimport { createSixbError } from \"@sixb/core/internal/errors\"\nimport { runStepLineage } from \"@sixb/core/internal/pipelines\"\nimport { type DatasetVersion, resolveStepLineage } from \"@sixb/core/lake-storage\"\nimport { throwIfAborted } from \"./errors\"\nimport { createStepInputs, type ResolvedStepInput } from \"./step-inputs\"\nimport type { PipelineJob, PipelineLogSession, PipelineWorkerContext } from \"./types\"\n\nexport async function executeRunStep(input: {\n readonly runtime: PipelineWorkerContext\n readonly pipeline: PipelineDefinition\n readonly step: PipelineStepDefinition\n readonly job: PipelineJob\n readonly signal: AbortSignal\n readonly logSession: PipelineLogSession\n readonly outputDataset: DatasetDefinition\n readonly expectedLatestVersionId: string | null\n readonly resolvedInputs: readonly ResolvedStepInput[]\n}): Promise<{\n readonly version: DatasetVersion\n readonly versionCreated: boolean\n readonly rowsWritten: number\n}> {\n const { runtime, pipeline, step, job, signal, outputDataset, resolvedInputs } = input\n\n if (step.executor.kind !== \"run\") {\n throw createSixbError(\n \"internal.unexpected\",\n `[SixbPipelineWorker] Pipeline '${pipeline.id}' step '${step.id}' uses SQL execution, which is not supported by the run step executor.`,\n {\n details: {\n pipelineId: pipeline.id,\n pipelineRunId: job.id,\n stepId: step.id,\n datasetId: outputDataset.id,\n },\n }\n )\n }\n\n const logger = input.logSession.withContext({ stepId: step.id })\n let rowsWritten = 0\n const write = await runtime.lakeStorage.beginWrite({\n dataset: outputDataset,\n mode: step.mode,\n inputs: resolvedInputs.map((resolved) => resolved.ref),\n producer: { kind: \"pipeline\", id: pipeline.id, runId: job.id, stepId: step.id },\n // Through every upstream hop: each column the step reads stands for what its input version\n // records. The output is the only registered dataset the step's own lineage reads.\n lineage: resolveStepLineage(\n runStepLineage(step, new Map([[outputDataset.id, outputDataset]])),\n resolvedInputs.map((resolved) => resolved.version)\n ),\n })\n\n try {\n const context: PipelineStepRunContext = {\n projectId: runtime.id,\n pipelineId: pipeline.id,\n stepId: step.id,\n runId: job.id,\n signal,\n logger,\n inputs: createStepInputs({\n lakeStorage: runtime.lakeStorage,\n pipeline,\n step,\n resolvedInputs,\n signal,\n }),\n output: {\n async writeRows(rows) {\n await write.writeRows(\n countRowsWritten(rows, signal, () => {\n rowsWritten += 1\n })\n )\n },\n },\n }\n\n await step.executor.handler(context)\n throwIfAborted(signal)\n\n const commit = await write.commit({\n expectedLatestVersionId: input.expectedLatestVersionId,\n commitMessage: `pipeline ${pipeline.id} step ${step.id} run ${job.id}`,\n })\n const { outcome, ...version } = commit\n\n return {\n version,\n versionCreated: outcome === \"created\",\n rowsWritten,\n }\n } catch (error) {\n await write.abort().catch(() => {})\n throw error\n }\n}\n\nasync function* countRowsWritten(\n rows: Iterable<DatasetRow> | AsyncIterable<DatasetRow>,\n signal: AbortSignal,\n onWritten: () => void\n): AsyncIterable<DatasetRow> {\n for await (const row of rows) {\n throwIfAborted(signal)\n yield row\n onWritten()\n }\n}\n",
11
+ "import type {\n DatasetDefinition,\n PipelineDefinition,\n PipelineStepDefinition,\n PipelineStepInput,\n PipelineStepRunContext,\n} from \"@sixb/core\"\nimport { createSixbError } from \"@sixb/core/internal/errors\"\nimport { isDatasetPick, stepInputColumns } from \"@sixb/core/internal/pipelines\"\nimport type {\n DatasetVersion,\n DatasetVersionRef,\n ReadDatasetRowsInput,\n} from \"@sixb/core/lake-storage\"\nimport { requireRegisteredDataset } from \"./errors\"\nimport type { PipelineWorkerContext } from \"./types\"\n\nexport interface ResolvedStepInput {\n readonly name: string\n /** The registered dataset. */\n readonly dataset: DatasetDefinition\n /** Present when the step input is `dataset.pick(...)`: the step reads only those columns. */\n readonly pick?: {\n readonly dataset: DatasetDefinition\n readonly columns: readonly string[]\n }\n readonly version: DatasetVersion\n readonly ref: DatasetVersionRef\n}\n\nexport async function resolveStepInputs(input: {\n readonly runtime: PipelineWorkerContext\n readonly pipeline: PipelineDefinition\n readonly step: PipelineStepDefinition\n readonly pipelineRunId: string\n}): Promise<readonly ResolvedStepInput[]> {\n const { runtime, pipeline, step, pipelineRunId } = input\n const resolved: ResolvedStepInput[] = []\n\n for (const [name, declaredDataset] of Object.entries(step.inputs)) {\n const dataset = requireRegisteredDataset({\n dataset: runtime.datasets.getById(declaredDataset.id),\n pipelineId: pipeline.id,\n pipelineRunId,\n stepId: step.id,\n role: \"input\",\n name,\n datasetId: declaredDataset.id,\n })\n\n const version = await runtime.lakeStorage.getLatestVersion(dataset.id)\n if (!version) {\n throw createSixbError(\n \"internal.unexpected\",\n `[SixbPipelineWorker] Pipeline '${pipeline.id}' step '${step.id}' input '${name}' dataset '${dataset.id}' has no committed version.`,\n {\n details: {\n pipelineId: pipeline.id,\n pipelineRunId,\n stepId: step.id,\n datasetId: dataset.id,\n },\n }\n )\n }\n\n resolved.push({\n name,\n dataset,\n ...(isDatasetPick(declaredDataset)\n ? { pick: { dataset: declaredDataset, columns: stepInputColumns(declaredDataset) } }\n : {}),\n version,\n ref: {\n datasetId: version.datasetId,\n versionId: version.versionId,\n },\n })\n }\n\n return resolved\n}\n\nexport function createStepInputs(input: {\n readonly lakeStorage: PipelineWorkerContext[\"lakeStorage\"]\n readonly pipeline: PipelineDefinition\n readonly step: PipelineStepDefinition\n readonly resolvedInputs: readonly ResolvedStepInput[]\n readonly signal: AbortSignal\n}): PipelineStepRunContext[\"inputs\"] {\n const { lakeStorage, signal } = input\n const inputs: Record<string, PipelineStepInput> = {}\n\n for (const resolved of input.resolvedInputs) {\n const subject = `[SixbPipelineWorker] Pipeline '${input.pipeline.id}' step '${input.step.id}' input '${resolved.name}'`\n const { pick } = resolved\n\n inputs[resolved.name] = {\n dataset: pick?.dataset ?? resolved.dataset,\n version: resolved.version,\n readRows(readInput: Omit<ReadDatasetRowsInput, \"datasetId\" | \"versionId\"> = {}) {\n // Step readers stay pinned even if newer versions commit while the handler runs.\n return lakeStorage.readRows({\n ...readInput,\n ...(pick && { columns: readPickedColumns(subject, pick.columns, readInput.columns) }),\n signal: readInput.signal ? AbortSignal.any([signal, readInput.signal]) : signal,\n datasetId: resolved.dataset.id,\n versionId: resolved.version.versionId,\n })\n },\n }\n }\n\n return inputs\n}\n\n/**\n * The version records that the step read only the picked columns, which keeps the markings of the\n * others out of its output. Reading another column would make that record false.\n */\nfunction readPickedColumns(\n subject: string,\n pickedColumns: readonly string[],\n requested: readonly string[] | undefined\n): readonly string[] {\n // Lake providers read every column for an empty list.\n if (requested === undefined || requested.length === 0) return pickedColumns\n\n const unpicked = requested.find((column) => !pickedColumns.includes(column))\n if (unpicked !== undefined) {\n throw new Error(\n `${subject} reads column '${unpicked}', which its pick leaves out. Add '${unpicked}' to the pick to read it.`\n )\n }\n\n return requested\n}\n",
12
+ "import type {\n DatasetDefinition,\n LakeStorage,\n LakeStorageWithSql,\n PipelineDefinition,\n PipelineStepDefinition,\n} from \"@sixb/core\"\nimport { createSixbError } from \"@sixb/core/internal/errors\"\nimport type {\n DatasetVersion,\n DatasetWriteMode,\n LakeSqlTransformCapabilities,\n} from \"@sixb/core/lake-storage\"\nimport { throwIfAborted } from \"./errors\"\nimport type { ResolvedStepInput } from \"./step-inputs\"\nimport type { PipelineJob, PipelineWorkerContext } from \"./types\"\n\nfunction hasSqlExecutor(lakeStorage: LakeStorage): lakeStorage is LakeStorageWithSql {\n const sql = \"sql\" in lakeStorage ? lakeStorage.sql : null\n\n return (\n typeof sql === \"object\" && sql !== null && \"execute\" in sql && typeof sql.execute === \"function\"\n )\n}\n\nfunction assertSqlModeSupported(input: {\n readonly capabilities: LakeSqlTransformCapabilities\n readonly dialect: string\n readonly mode: DatasetWriteMode\n readonly pipelineId: string\n readonly pipelineRunId: string\n readonly stepId: string\n readonly datasetId: string\n}): void {\n const supported =\n input.mode === \"append\"\n ? input.capabilities.supportsAppend\n : input.capabilities.supportsSnapshot\n if (supported) return\n\n throw createSixbError(\n \"internal.unexpected\",\n `[SixbPipelineWorker] Pipeline '${input.pipelineId}' step '${input.stepId}' writes in ` +\n `'${input.mode}' mode, which the ${input.dialect} SQL executor does not support.`,\n {\n details: {\n pipelineId: input.pipelineId,\n pipelineRunId: input.pipelineRunId,\n stepId: input.stepId,\n datasetId: input.datasetId,\n },\n }\n )\n}\n\nexport async function executeSqlStep(input: {\n readonly runtime: PipelineWorkerContext\n readonly pipeline: PipelineDefinition\n readonly step: PipelineStepDefinition\n readonly job: PipelineJob\n readonly signal: AbortSignal\n readonly outputDataset: DatasetDefinition\n readonly expectedLatestVersionId: string | null\n readonly resolvedInputs: readonly ResolvedStepInput[]\n}): Promise<{\n readonly version: DatasetVersion\n readonly versionCreated: boolean\n readonly rowsWritten?: number\n}> {\n const { runtime, pipeline, step, job, signal, outputDataset, resolvedInputs } = input\n\n if (step.executor.kind !== \"sql\") {\n throw createSixbError(\n \"internal.unexpected\",\n `[SixbPipelineWorker] Pipeline '${pipeline.id}' step '${step.id}' uses run execution, which is not supported by the SQL step executor.`,\n {\n details: {\n pipelineId: pipeline.id,\n pipelineRunId: job.id,\n stepId: step.id,\n datasetId: outputDataset.id,\n },\n }\n )\n }\n\n if (!hasSqlExecutor(runtime.lakeStorage)) {\n throw createSixbError(\n \"internal.unexpected\",\n `[SixbPipelineWorker] Pipeline '${pipeline.id}' step '${step.id}' requires SQL transform support, but lake storage does not provide lakeStorage.sql.execute(...).`,\n {\n details: {\n pipelineId: pipeline.id,\n pipelineRunId: job.id,\n stepId: step.id,\n datasetId: outputDataset.id,\n },\n }\n )\n }\n\n // `LakeSqlExecutor.capabilities` is a required field on the contract that nothing read,\n // so a provider declaring `supportsAppend: false` was asked to be honest and then\n // ignored — the unsupported mode failed somewhere inside the provider instead, in\n // whatever words that provider happened to use.\n assertSqlModeSupported({\n capabilities: runtime.lakeStorage.sql.capabilities,\n dialect: runtime.lakeStorage.sql.dialect,\n mode: step.mode,\n pipelineId: pipeline.id,\n pipelineRunId: job.id,\n stepId: step.id,\n datasetId: outputDataset.id,\n })\n\n throwIfAborted(signal)\n\n const commit = await runtime.lakeStorage.sql.execute({\n sources: Object.fromEntries(\n resolvedInputs.map((input) => [\n input.name,\n {\n dataset: input.dataset,\n versionId: input.version.versionId,\n ...(input.pick && { columns: input.pick.columns }),\n },\n ])\n ),\n sql: step.executor.sql,\n target: outputDataset,\n expectedLatestVersionId: input.expectedLatestVersionId,\n mode: step.mode,\n producer: {\n kind: \"pipeline\",\n id: pipeline.id,\n runId: job.id,\n stepId: step.id,\n },\n commitMessage: `pipeline ${pipeline.id} step ${step.id} run ${job.id}`,\n })\n\n throwIfAborted(signal)\n const { outcome, ...version } = commit\n\n return {\n version,\n versionCreated: outcome === \"created\",\n rowsWritten: version.rowCount,\n }\n}\n"
13
13
  ],
14
- "mappings": ";;AAAA;AACA;AAEA;AAEA,uCAAS;;;ACGT,IAAM,SAAS;AAEf,eAAsB,sBAAsB,CAC1C,QACA,KACA,eACe;AAAA,EACf,IAAI,CAAC,IAAI,WAAW;AAAA,IAClB,MAAM,IAAI,MAAM,sCAAsC,IAAI,kCAAkC;AAAA,EAC9F;AAAA,EACA,MAAM,QAAQ,KACZ;AAAA,IACE,QAAQ;AAAA,MACN;AAAA,QACE,MAAM;AAAA,QACN,SAAS;AAAA,UACP,YAAY,IAAI;AAAA,UAChB,OAAO,IAAI;AAAA,UACX,WAAW,IAAI,UAAU,YAAY;AAAA,QACvC;AAAA,MACF;AAAA,IACF;AAAA,IACA;AAAA,EACF,GACA,EAAE,QAAQ,OAAO,CACnB;AAAA;AAGF,eAAsB,0BAA0B,CAC9C,QACA,MAIA,SACA,eACe;AAAA,EACf,MAAM,QAAQ,KACZ;AAAA,IACE,QAAQ;AAAA,MACN;AAAA,QACE,MAAM;AAAA,QACN,SAAS;AAAA,UACP,YAAY,KAAK;AAAA,UACjB,OAAO,KAAK;AAAA,UACZ,WAAW,KAAK;AAAA,UAChB,QAAQ,KAAK;AAAA,UACb,WAAW,QAAQ;AAAA,UACnB,YAAY,QAAQ;AAAA,UACpB,WAAW,KAAK;AAAA,UAChB,WAAW,KAAK,UAAU,YAAY;AAAA,QACxC;AAAA,MACF;AAAA,IACF;AAAA,IACA;AAAA,EACF,GACA,EAAE,QAAQ,OAAO,CACnB;AAAA;AAGF,eAAsB,2BAA2B,CAC/C,QACA,MAaA,SACA,eACe;AAAA,EACf,MAAM,QAAQ,KACZ;AAAA,IACE,QAAQ;AAAA,MACN;AAAA,QACE,MAAM;AAAA,QACN,SAAS;AAAA,UACP,YAAY,KAAK;AAAA,UACjB,OAAO,KAAK;AAAA,UACZ,WAAW,KAAK;AAAA,UAChB,QAAQ,KAAK;AAAA,UACb,WAAW,QAAQ;AAAA,UACnB,YAAY,QAAQ;AAAA,UACpB,WAAW,KAAK;AAAA,UAChB,QAAQ,sBAAsB,KAAK,QAAQ,sBAAsB,KAAK,KAAK;AAAA,UAC3E,YAAY,kBAAkB,KAAK,IAAI,KAAK,UAAU,EAAE,YAAY;AAAA,aAChE,KAAK,SAAS,EAAE,WAAW,KAAK,OAAO,UAAU,IAAI,CAAC;AAAA,aACtD,KAAK,gBAAgB,YAAY,EAAE,aAAa,KAAK,YAAY,IAAI,CAAC;AAAA,aACtE,KAAK,QAAQ,EAAE,OAAO,KAAK,MAAM,IAAI,CAAC;AAAA,QAC5C;AAAA,MACF;AAAA,IACF;AAAA,IACA;AAAA,EACF,GACA,EAAE,QAAQ,OAAO,CACnB;AAAA;AAGF,eAAsB,2BAA2B,CAC/C,QACA,KACA,MACA,eACe;AAAA,EACf,IAAI,CAAC,KAAK;AAAA,IAAgB;AAAA,EAG1B,MAAM,QAAQ,KACZ;AAAA,IACE,QAAQ;AAAA,MACN;AAAA,QACE,MAAM;AAAA,QACN,SAAS;AAAA,UACP,WAAW,KAAK,QAAQ;AAAA,UACxB,WAAW,KAAK,QAAQ;AAAA,UACxB,WAAW,KAAK,QAAQ,UAAU,YAAY;AAAA,UAC9C,UAAU;AAAA,YACR,MAAM;AAAA,YACN,IAAI,IAAI;AAAA,YACR,OAAO,IAAI;AAAA,YACX,QAAQ,KAAK,IAAI;AAAA,UACnB;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,IACA;AAAA,EACF,GACA,EAAE,QAAQ,OAAO,CACnB;AAAA;AAGF,eAAsB,uBAAuB,CAC3C,QACA,KACA,eACe;AAAA,EACf,MAAM,QAAQ,KACZ;AAAA,IACE,QAAQ,CAAC,EAAE,MAAM,yBAAyB,SAAS,gCAAgC,GAAG,EAAE,CAAC;AAAA,IACzF;AAAA,EACF,GACA,EAAE,QAAQ,OAAO,CACnB;AAAA;AAGF,SAAS,+BAA+B,CACtC,KACA;AAAA,EACA,OAAO;AAAA,IACL,YAAY,IAAI;AAAA,IAChB,OAAO,IAAI;AAAA,IACX,QAAQ,sBAAsB,IAAI,QAAQ,iBAAiB,IAAI,KAAK;AAAA,OAChE,IAAI,SACJ;AAAA,MACE,WAAW,IAAI,OAAO;AAAA,MACtB,WAAW,IAAI,OAAO;AAAA,IACxB,IACA,CAAC;AAAA,OACD,IAAI,QAAQ,EAAE,OAAO,IAAI,MAAM,IAAI,CAAC;AAAA,EAC1C;AAAA;AAGF,SAAS,iBAAiB,CAAC,WAAmB,YAAoC;AAAA,EAChF,IAAI,YAAY;AAAA,IACd,OAAO;AAAA,EACT;AAAA,EAEA,MAAM,IAAI,MAAM,2CAA2C,+BAA+B;AAAA;AAG5F,SAAS,qBAAqB,CAC5B,QACA,SACkD;AAAA,EAClD,IAAI,WAAW,YAAY,WAAW,WAAW;AAAA,IAC/C,MAAM,IAAI,MAAM,wBAAwB,2BAA2B;AAAA,EACrE;AAAA,EAEA,OAAO;AAAA;;;ACjMT,+BAAS;AACT;AAEA,uCAAS;;;ACFT;AAKO,SAAS,cAAc,CAAC,QAA2B;AAAA,EACxD,IAAI,OAAO,SAAS;AAAA,IAClB,MAAM,iBAAiB;AAAA,EACzB;AAAA;AAGK,SAAS,gBAAgB,CAC9B,QACA,OACoD;AAAA,EACpD,OAAO,OAAO,WAAW,aAAa,KAAK,IAAI,cAAc;AAAA;AAGxD,SAAS,kBAAiB,CAAC,OAIzB;AAAA,EACP,IAAI,MAAM,YAAY;AAAA,IACpB,OAAO,MAAM;AAAA,EACf;AAAA,EAEA,MAAM,gBACJ,uBACA,sCAAsC,MAAM,mDAC5C,EAAE,SAAS,EAAE,YAAY,MAAM,YAAY,OAAO,MAAM,MAAM,EAAE,CAClE;AAAA;AAGK,SAAS,0BAA0B,CAAC,SAOjC;AAAA,EAER,OAAO,gBACL,uBACA,kCAAkC,QAAQ,qBAAqB,QAAQ,sCAAsC,QAAQ,QAAQ,gDAAgD,QAAQ,sGACrL;AAAA,IACE,OAAO,QAAQ;AAAA,IACf,SAAS;AAAA,MACP,YAAY,QAAQ;AAAA,MACpB,eAAe,QAAQ;AAAA,MACvB,QAAQ,QAAQ;AAAA,MAChB,WAAW,QAAQ;AAAA,MACnB,WAAW,QAAQ,QAAQ;AAAA,MAC3B,WAAW,QAAQ,QAAQ;AAAA,IAC7B;AAAA,EACF,CACF;AAAA;AAGK,SAAS,yBAAyB,CAAC,SAMvC;AAAA,EACD,OAAO,gBACL,wBACA,sBAAsB,QAAQ,OAAO,iCAAiC,GACtE;AAAA,IACE,OAAO,QAAQ;AAAA,IACf,SAAS;AAAA,MACP,YAAY,QAAQ;AAAA,MACpB,eAAe,QAAQ;AAAA,MACvB,QAAQ,QAAQ;AAAA,SACZ,QAAQ,YAAY,EAAE,WAAW,QAAQ,UAAU,IAAI,CAAC;AAAA,IAC9D;AAAA,EACF,CACF;AAAA;AAGK,SAAS,yBAAyB,CAAC,OAAyB;AAAA,EACjE,OAAO,YAAY,KAAK,KAAK,MAAM,SAAS,0BAA0B,MAAM,UAAU,YAClF,MAAM,QACN;AAAA;AAGC,SAAS,8BAA8B,CAAC,SAKrC;AAAA,EACR,OAAO,gBACL,uBACA,kCAAkC,QAAQ,gDAAgD,QAAQ,QAAQ,oDAAoD,QAAQ,sGACtK;AAAA,IACE,OAAO,QAAQ;AAAA,IACf,SAAS;AAAA,MACP,YAAY,QAAQ;AAAA,MACpB,OAAO,QAAQ;AAAA,MACf,WAAW,QAAQ,QAAQ;AAAA,MAC3B,WAAW,QAAQ,QAAQ;AAAA,IAC7B;AAAA,EACF,CACF;AAAA;AAGK,SAAS,eAAe,CAC7B,UACA,KACoB;AAAA,EACpB,IAAI,CAAC,UAAU;AAAA,IACb,MAAM,gBACJ,uBACA,0CAA0C,IAAI,gBAC9C,EAAE,SAAS,EAAE,YAAY,IAAI,YAAY,OAAO,IAAI,GAAG,EAAE,CAC3D;AAAA,EACF;AAAA,EAEA,OAAO;AAAA;AAGF,SAAS,wBAAwB,CAAC,SAQnB;AAAA,EACpB,IAAI,QAAQ,SAAS;AAAA,IACnB,OAAO,QAAQ;AAAA,EACjB;AAAA,EAEA,MAAM,YAAY,QAAQ,SAAS,UAAU,UAAU,QAAQ,QAAQ,eAAe;AAAA,EAEtF,MAAM,gBACJ,uBACA,kCAAkC,QAAQ,qBAAqB,QAAQ,WAAW,yCAAyC,QAAQ,eACnI;AAAA,IACE,SAAS;AAAA,MACP,YAAY,QAAQ;AAAA,MACpB,eAAe,QAAQ;AAAA,MACvB,QAAQ,QAAQ;AAAA,MAChB,WAAW,QAAQ;AAAA,IACrB;AAAA,EACF,CACF;AAAA;AAGF,SAAS,gBAAgB,GAAU;AAAA,EACjC,MAAM,QAAQ,IAAI,MAAM,yBAAyB;AAAA,EACjD,MAAM,OAAO;AAAA,EACb,OAAO;AAAA;AAGT,SAAS,YAAY,CAAC,OAAyB;AAAA,EAC7C,OAAO,iBAAiB,SAAS,MAAM,SAAS;AAAA;;;AChKlD;AACA;AAEA;;;ACGA,4BAAS;;;ACAT,4BAAS;AAgBT,eAAsB,iBAAiB,CAAC,OAKE;AAAA,EACxC,QAAQ,SAAS,UAAU,MAAM,kBAAkB;AAAA,EACnD,MAAM,WAAgC,CAAC;AAAA,EAEvC,YAAY,MAAM,oBAAoB,OAAO,QAAQ,KAAK,MAAM,GAAG;AAAA,IACjE,MAAM,UAAU,yBAAyB;AAAA,MACvC,SAAS,QAAQ,SAAS,QAAQ,gBAAgB,EAAE;AAAA,MACpD,YAAY,SAAS;AAAA,MACrB;AAAA,MACA,QAAQ,KAAK;AAAA,MACb,MAAM;AAAA,MACN;AAAA,MACA,WAAW,gBAAgB;AAAA,IAC7B,CAAC;AAAA,IAED,MAAM,UAAU,MAAM,QAAQ,YAAY,iBAAiB,QAAQ,EAAE;AAAA,IACrE,IAAI,CAAC,SAAS;AAAA,MACZ,MAAM,iBACJ,uBACA,kCAAkC,SAAS,aAAa,KAAK,cAAc,kBAAkB,QAAQ,iCACrG;AAAA,QACE,SAAS;AAAA,UACP,YAAY,SAAS;AAAA,UACrB;AAAA,UACA,QAAQ,KAAK;AAAA,UACb,WAAW,QAAQ;AAAA,QACrB;AAAA,MACF,CACF;AAAA,IACF;AAAA,IAEA,SAAS,KAAK;AAAA,MACZ;AAAA,MACA;AAAA,MACA;AAAA,MACA,KAAK;AAAA,QACH,WAAW,QAAQ;AAAA,QACnB,WAAW,QAAQ;AAAA,MACrB;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,OAAO;AAAA;AAGF,SAAS,gBAAgB,CAC9B,aACA,UACA,QACkC;AAAA,EAClC,MAAM,SAA4C,CAAC;AAAA,EAEnD,WAAW,SAAS,UAAU;AAAA,IAC5B,OAAO,MAAM,QAAQ;AAAA,MACnB,SAAS,MAAM;AAAA,MACf,SAAS,MAAM;AAAA,MACf,QAAQ,CAAC,YAAmE,CAAC,GAAG;AAAA,QAE9E,OAAO,YAAY,SAAS;AAAA,aACvB;AAAA,UACH,QAAQ,UAAU,SAAS,YAAY,IAAI,CAAC,QAAQ,UAAU,MAAM,CAAC,IAAI;AAAA,UACzE,WAAW,MAAM,QAAQ;AAAA,UACzB,WAAW,MAAM,QAAQ;AAAA,QAC3B,CAAC;AAAA;AAAA,IAEL;AAAA,EACF;AAAA,EAEA,OAAO;AAAA;;;ADnFT,eAAsB,cAAc,CAAC,OAclC;AAAA,EACD,QAAQ,SAAS,UAAU,MAAM,KAAK,QAAQ,eAAe,mBAAmB;AAAA,EAEhF,IAAI,KAAK,SAAS,SAAS,OAAO;AAAA,IAChC,MAAM,iBACJ,uBACA,kCAAkC,SAAS,aAAa,KAAK,4EAC7D;AAAA,MACE,SAAS;AAAA,QACP,YAAY,SAAS;AAAA,QACrB,eAAe,IAAI;AAAA,QACnB,QAAQ,KAAK;AAAA,QACb,WAAW,cAAc;AAAA,MAC3B;AAAA,IACF,CACF;AAAA,EACF;AAAA,EAEA,MAAM,SAAS,MAAM,WAAW,YAAY,EAAE,QAAQ,KAAK,GAAG,CAAC;AAAA,EAC/D,IAAI,cAAc;AAAA,EAClB,MAAM,QAAQ,MAAM,QAAQ,YAAY,WAAW;AAAA,IACjD,SAAS;AAAA,IACT,MAAM,KAAK;AAAA,IACX,QAAQ,eAAe,IAAI,CAAC,aAAa,SAAS,GAAG;AAAA,IACrD,UAAU;AAAA,MACR,MAAM;AAAA,MACN,IAAI,SAAS;AAAA,MACb,OAAO,IAAI;AAAA,MACX,QAAQ,KAAK;AAAA,IACf;AAAA,EACF,CAAC;AAAA,EAED,IAAI;AAAA,IACF,MAAM,UAAkC;AAAA,MACtC,WAAW,QAAQ;AAAA,MACnB,YAAY,SAAS;AAAA,MACrB,QAAQ,KAAK;AAAA,MACb,OAAO,IAAI;AAAA,MACX;AAAA,MACA;AAAA,MACA,QAAQ,iBAAiB,QAAQ,aAAa,gBAAgB,MAAM;AAAA,MACpE,QAAQ;AAAA,aACA,UAAS,CAAC,MAAM;AAAA,UACpB,MAAM,MAAM,UACV,iBAAiB,MAAM,QAAQ,MAAM;AAAA,YACnC,eAAe;AAAA,WAChB,CACH;AAAA;AAAA,MAEJ;AAAA,IACF;AAAA,IAEA,MAAM,KAAK,SAAS,QAAQ,OAAO;AAAA,IACnC,eAAe,MAAM;AAAA,IAErB,MAAM,SAAS,MAAM,MAAM,OAAO;AAAA,MAChC,yBAAyB,MAAM;AAAA,MAC/B,eAAe,YAAY,SAAS,WAAW,KAAK,UAAU,IAAI;AAAA,IACpE,CAAC;AAAA,IACD,QAAQ,YAAY,YAAY;AAAA,IAEhC,OAAO;AAAA,MACL;AAAA,MACA,gBAAgB,YAAY;AAAA,MAC5B;AAAA,IACF;AAAA,IACA,OAAO,OAAO;AAAA,IACd,MAAM,MAAM,MAAM,EAAE,MAAM,MAAM,EAAE;AAAA,IAClC,MAAM;AAAA;AAAA;AAIV,gBAAgB,gBAAgB,CAC9B,MACA,QACA,WAC2B;AAAA,EAC3B,iBAAiB,OAAO,MAAM;AAAA,IAC5B,eAAe,MAAM;AAAA,IACrB,MAAM;AAAA,IACN,UAAU;AAAA,EACZ;AAAA;;;AErGF,4BAAS;AAUT,SAAS,cAAc,CAAC,aAA6D;AAAA,EACnF,MAAM,MAAM,SAAS,cAAc,YAAY,MAAM;AAAA,EAErD,OACE,OAAO,QAAQ,YAAY,QAAQ,QAAQ,aAAa,OAAO,OAAO,IAAI,YAAY;AAAA;AAI1F,SAAS,sBAAsB,CAAC,OAQvB;AAAA,EACP,MAAM,YACJ,MAAM,SAAS,WACX,MAAM,aAAa,iBACnB,MAAM,aAAa;AAAA,EACzB,IAAI;AAAA,IAAW;AAAA,EAEf,MAAM,iBACJ,uBACA,kCAAkC,MAAM,qBAAqB,MAAM,uBACjE,IAAI,MAAM,yBAAyB,MAAM,0CAC3C;AAAA,IACE,SAAS;AAAA,MACP,YAAY,MAAM;AAAA,MAClB,eAAe,MAAM;AAAA,MACrB,QAAQ,MAAM;AAAA,MACd,WAAW,MAAM;AAAA,IACnB;AAAA,EACF,CACF;AAAA;AAGF,eAAsB,cAAc,CAAC,OAalC;AAAA,EACD,QAAQ,SAAS,UAAU,MAAM,KAAK,QAAQ,eAAe,mBAAmB;AAAA,EAEhF,IAAI,KAAK,SAAS,SAAS,OAAO;AAAA,IAChC,MAAM,iBACJ,uBACA,kCAAkC,SAAS,aAAa,KAAK,4EAC7D;AAAA,MACE,SAAS;AAAA,QACP,YAAY,SAAS;AAAA,QACrB,eAAe,IAAI;AAAA,QACnB,QAAQ,KAAK;AAAA,QACb,WAAW,cAAc;AAAA,MAC3B;AAAA,IACF,CACF;AAAA,EACF;AAAA,EAEA,IAAI,CAAC,eAAe,QAAQ,WAAW,GAAG;AAAA,IACxC,MAAM,iBACJ,uBACA,kCAAkC,SAAS,aAAa,KAAK,uGAC7D;AAAA,MACE,SAAS;AAAA,QACP,YAAY,SAAS;AAAA,QACrB,eAAe,IAAI;AAAA,QACnB,QAAQ,KAAK;AAAA,QACb,WAAW,cAAc;AAAA,MAC3B;AAAA,IACF,CACF;AAAA,EACF;AAAA,EAMA,uBAAuB;AAAA,IACrB,cAAc,QAAQ,YAAY,IAAI;AAAA,IACtC,SAAS,QAAQ,YAAY,IAAI;AAAA,IACjC,MAAM,KAAK;AAAA,IACX,YAAY,SAAS;AAAA,IACrB,eAAe,IAAI;AAAA,IACnB,QAAQ,KAAK;AAAA,IACb,WAAW,cAAc;AAAA,EAC3B,CAAC;AAAA,EAED,eAAe,MAAM;AAAA,EAErB,MAAM,SAAS,MAAM,QAAQ,YAAY,IAAI,QAAQ;AAAA,IACnD,SAAS,OAAO,YACd,eAAe,IAAI,CAAC,UAAU;AAAA,MAC5B,MAAM;AAAA,MACN;AAAA,QACE,SAAS,MAAM;AAAA,QACf,WAAW,MAAM,QAAQ;AAAA,MAC3B;AAAA,IACF,CAAC,CACH;AAAA,IACA,KAAK,KAAK,SAAS;AAAA,IACnB,QAAQ;AAAA,IACR,yBAAyB,MAAM;AAAA,IAC/B,MAAM,KAAK;AAAA,IACX,UAAU;AAAA,MACR,MAAM;AAAA,MACN,IAAI,SAAS;AAAA,MACb,OAAO,IAAI;AAAA,MACX,QAAQ,KAAK;AAAA,IACf;AAAA,IACA,eAAe,YAAY,SAAS,WAAW,KAAK,UAAU,IAAI;AAAA,EACpE,CAAC;AAAA,EAED,eAAe,MAAM;AAAA,EACrB,QAAQ,YAAY,YAAY;AAAA,EAEhC,OAAO;AAAA,IACL;AAAA,IACA,gBAAgB,YAAY;AAAA,IAC5B,aAAa,QAAQ;AAAA,EACvB;AAAA;;;AHrHF,eAAsB,OAAO,CAAC,OAUK;AAAA,EACjC,QAAQ,SAAS,UAAU,MAAM,WAAW,KAAK,WAAW;AAAA,EAC5D,MAAM,mBAAmB;AAAA,IACvB;AAAA,IACA,YAAY,SAAS,MAAM,MAAM;AAAA,EACnC;AAAA,EAEA,eAAe,MAAM;AAAA,EACrB,IAAI;AAAA,EACJ,IAAI;AAAA,EACJ,IAAI;AAAA,EACJ,IAAI;AAAA,IACF,gBAAgB,sBACd,yBAAyB;AAAA,MACvB,SAAS,QAAQ,SAAS,QAAQ,KAAK,OAAO,EAAE;AAAA,MAChD,YAAY,SAAS;AAAA,MACrB,eAAe,IAAI;AAAA,MACnB,QAAQ,KAAK;AAAA,MACb,MAAM;AAAA,MACN,WAAW,KAAK,OAAO;AAAA,IACzB,CAAC,CACH;AAAA,IAEA,MAAM,QAAQ,YAAY,cAAc,aAAa;AAAA,IAGrD,2BACG,MAAM,QAAQ,YAAY,iBAAiB,cAAc,EAAE,IAAI,aAAa;AAAA,IAC/E,iBAAiB,MAAM,kBAAkB;AAAA,MACvC;AAAA,MACA;AAAA,MACA;AAAA,MACA,eAAe,IAAI;AAAA,IACrB,CAAC;AAAA,IACD,OAAO,OAAO;AAAA,IACd,IAAI,iBAAiB,QAAQ,KAAK,MAAM,UAAU;AAAA,MAChD,MAAM,0BAA0B;AAAA,QAC9B,YAAY,SAAS;AAAA,QACrB,eAAe,IAAI;AAAA,QACnB,QAAQ,KAAK;AAAA,QACb,OAAO;AAAA,MACT,CAAC;AAAA,IACH;AAAA,IACA,MAAM;AAAA;AAAA,EAER,MAAM,YAAY,eAAe,IAAI,CAAC,aAAa,SAAS,GAAG;AAAA,EAE/D,MAAM,YAAY,GAAG,IAAI,WAAW,YAAY,KAAK,KAAK;AAAA,EAC1D,MAAM,iBAAiB,MAAM,QAAQ,oBAAoB,UAAU;AAAA,IACjE,WAAW,QAAQ;AAAA,IACnB,IAAI;AAAA,IACJ,eAAe,IAAI;AAAA,IACnB,YAAY,SAAS;AAAA,IACrB,QAAQ,KAAK;AAAA,IACb,WAAW,cAAc;AAAA,IACzB,MAAM,KAAK;AAAA,IACX,QAAQ;AAAA,EACV,CAAC;AAAA,EACD,MAAM,MAAM,gBAAgB,gBAAgB,gBAAgB;AAAA,EAE5D,IAAI;AAAA,EACJ,IAAI;AAAA,EAEJ,IAAI;AAAA,IACF,MAAM,YAAY,MAAM,YAAY;AAAA,MAClC;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,YAAY,MAAM;AAAA,MAClB;AAAA,MACA;AAAA,MACA;AAAA,IACF,CAAC;AAAA,IACD,mBAAmB,UAAU;AAAA,IAC7B,cAAc,UAAU;AAAA,IAExB,IAAI;AAAA,IACJ,IAAI;AAAA,MACF,UAAU,MAAM,QAAQ,oBAAoB,WAAW;AAAA,QACrD,WAAW,QAAQ;AAAA,QACnB,IAAI;AAAA,QACJ,QAAQ;AAAA,QACR,QAAQ;AAAA,UACN,WAAW,iBAAiB;AAAA,UAC5B,WAAW,iBAAiB;AAAA,QAC9B;AAAA,QACA;AAAA,MACF,CAAC;AAAA,MACD,OAAO,OAAO;AAAA,MACd,MAAM,2BAA2B;AAAA,QAC/B,YAAY,SAAS;AAAA,QACrB,eAAe,IAAI;AAAA,QACnB,QAAQ,KAAK;AAAA,QACb;AAAA,QACA,SAAS;AAAA,QACT,OAAO;AAAA,MACT,CAAC;AAAA;AAAA,IAEH,MAAM,MAAM,iBAAiB,SAAS,gBAAgB;AAAA,IAEtD,OAAO;AAAA,MACL,KAAK;AAAA,MACL,SAAS;AAAA,MACT,gBAAgB,UAAU;AAAA,IAC5B;AAAA,IACA,OAAO,OAAO;AAAA,IACd,IAAI,CAAC,kBAAkB;AAAA,MACrB,MAAM,SAAS,iBAAiB,QAAQ,KAAK;AAAA,MAC7C,MAAM,eACJ,WAAW,WACP,0BAA0B;AAAA,QACxB,YAAY,SAAS;AAAA,QACrB,eAAe,IAAI;AAAA,QACnB,QAAQ,KAAK;AAAA,QACb;AAAA,QACA,OAAO;AAAA,MACT,CAAC,IACD;AAAA,MACN,MAAM,gBAAgB,MAAM,QAAQ,oBACjC,WAAW;AAAA,QACV,WAAW,QAAQ;AAAA,QACnB,IAAI;AAAA,QACJ;AAAA,QACA;AAAA,QACA,OAAO,mBAAmB,cAAc;AAAA,UACtC,cAAc;AAAA,UACd,aAAa,WAAW,cAAc,sBAAsB;AAAA,UAC5D,SAAS;AAAA,YACP,YAAY,SAAS;AAAA,YACrB,eAAe,IAAI;AAAA,YACnB,QAAQ,KAAK;AAAA,YACb;AAAA,UACF;AAAA,QACF,CAAC;AAAA,MACH,CAAC,EACA,MAAM,MAAM,IAAI;AAAA,MAEnB,IAAI,eAAe;AAAA,QACjB,MAAM,MAAM,iBAAiB,eAAe,gBAAgB;AAAA,MAC9D;AAAA,MAEA,MAAM;AAAA,IACR;AAAA,IAEA,MAAM;AAAA;AAAA;AAIV,eAAe,WAAW,CAAC,OAUM;AAAA,EAC/B,QAAQ,MAAM,KAAK,SAAS;AAAA,SACrB;AAAA,MACH,OAAO,eAAe,KAAK;AAAA,SACxB;AAAA,MACH,OAAO,eAAe,KAAK;AAAA;AAAA;;;AFzL1B,MAAM,uCAAuC,MAAM;AAAA,EAGnC;AAAA,EAFH,OAAO;AAAA,EAEzB,WAAW,CAAU,KAAwB;AAAA,IAC3C,MAAM,sCAAsC,IAAI,0BAA0B;AAAA,IADvD;AAAA;AAGvB;AAEA,eAAsB,cAAc,CAAC,OAAwD;AAAA,EAC3F,QAAQ,SAAS,KAAK,eAAe;AAAA,EACrC,MAAM,MAAM,EAAE,IAAI,WAAW,IAAI,YAAY,WAAW,WAAW;AAAA,EACnE,MAAM,SAAS,MAAM,UAAU,IAAI,gBAAgB,EAAE;AAAA,EACrD,IAAI,WAAW,cAAc,QAAQ,IAAI;AAAA,IACvC,MAAM,IAAI,MACR,sCAAsC,WAAW,2BAA2B,WAAW,oBAAoB,QAAQ,MACrH;AAAA,EACF;AAAA,EACA,MAAM,aAAa,sBAAsB,QAAQ,IAAI,QAAQ,OAAO,EAAE,eAAe;AAAA,IACnF,MAAM;AAAA,IACN,IAAI,IAAI;AAAA,EACV,CAAC;AAAA,EACD,MAAM,QAAiC,CAAC;AAAA,EACxC,IAAI;AAAA,EACJ,IAAI;AAAA,EACJ,IAAI,WAAW;AAAA,EAEf,eAAe,MAAM;AAAA,EAErB,IAAI;AAAA,IACF,IAAI;AAAA,MACF,aAAa,MAAM,QAAQ,oBAAoB,MAAM;AAAA,QACnD,WAAW,QAAQ;AAAA,QACnB,IAAI,IAAI;AAAA,MACV,CAAC;AAAA,MACD,OAAO,OAAO;AAAA,MACd,MAAM,WAAW,MAAM,QAAQ,oBAAoB,QAAQ;AAAA,QACzD,WAAW,QAAQ;AAAA,QACnB,IAAI,IAAI;AAAA,MACV,CAAC;AAAA,MACD,IACE,UAAU,eAAe,WAAW,cACpC,SAAS,gBAAgB,WAAW,aACpC;AAAA,QACA,MAAM,IAAI,+BAA+B,QAAQ;AAAA,MACnD;AAAA,MACA,MAAM;AAAA;AAAA,IAER,MAAM,MAAM,eAAe,UAAU;AAAA,IAErC,MAAM,WAAW,gBAAgB,QAAQ,UAAU,QAAQ,IAAI,UAAU,GAAG,GAAG;AAAA,IAE/E,YAAY,WAAW,SAAS,SAAS,MAAM,MAAM,QAAQ,GAAG;AAAA,MAC9D,MAAM,aAAa,MAAM,QAAQ;AAAA,QAC/B;AAAA,QACA;AAAA,QACA,MAAM,KAAK;AAAA,QACX;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA,eAAe,MAAM;AAAA,QACrB,gBAAgB,MAAM;AAAA,MACxB,CAAC;AAAA,MAED,MAAM,KAAK,UAAU;AAAA,MACrB,eAAe,WAAW;AAAA,MAE1B,MAAM,MAAM,kBAAkB,UAAU;AAAA,IAC1C;AAAA,IAEA,IAAI;AAAA,IACJ,IAAI;AAAA,MACF,MAAM,MAAM,QAAQ,oBAAoB,OAAO;AAAA,QAC7C,WAAW,QAAQ;AAAA,QACnB,IAAI,IAAI;AAAA,QACR,QAAQ;AAAA,QACR,QAAQ,eACJ;AAAA,UACE,WAAW,aAAa;AAAA,UACxB,WAAW,aAAa;AAAA,QAC1B,IACA;AAAA,MACN,CAAC;AAAA,MACD,OAAO,OAAO;AAAA,MACd,IAAI,cAAc;AAAA,QAChB,MAAM,+BAA+B;AAAA,UACnC,YAAY,SAAS;AAAA,UACrB,OAAO,IAAI;AAAA,UACX,SAAS;AAAA,UACT,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AAAA,MACA,MAAM;AAAA;AAAA,IAER,MAAM,cAAc;AAAA,SACf;AAAA,MACH,YAAY,mBAAkB;AAAA,QAC5B,YAAY,SAAS;AAAA,QACrB,OAAO,IAAI;AAAA,QACX,YAAY,IAAI;AAAA,MAClB,CAAC;AAAA,IACH;AAAA,IAEA,WAAW;AAAA,IACX,MAAM,kBAAkB,MAAM,eAAe,WAAW;AAAA,IACxD,OAAO;AAAA,MACL,KAAK;AAAA,MACL;AAAA,MACA,SAAS;AAAA,IACX;AAAA,IACA,OAAO,OAAO;AAAA,IACd,MAAM,gBAAgB,0BAA0B,KAAK;AAAA,IACrD,IAAI,cAAc,CAAC,UAAU;AAAA,MAC3B,MAAM,SAAS,iBAAiB,QAAQ,KAAK;AAAA,MAC7C,IAAI;AAAA,QACF,MAAM,UAAU,oBAAmB,OAAO;AAAA,UACxC,cAAc;AAAA,UACd,aAAa,WAAW,cAAc,sBAAsB;AAAA,UAC5D,SAAS,EAAE,YAAY,WAAW,YAAY,OAAO,IAAI,GAAG;AAAA,QAC9D,CAAC;AAAA,QACD,MAAM,MAAM,MAAM,QAAQ,oBAAoB,OAAO;AAAA,UACnD,WAAW,QAAQ;AAAA,UACnB,IAAI,IAAI;AAAA,UACR;AAAA,UACA,OAAO;AAAA,QACT,CAAC;AAAA,QACD,IAAI,WAAW,YAAY,IAAI,WAAW,UAAU;AAAA,UAClD,MAAM,cAAc,eAAe,KAAK,OAAO;AAAA,QACjD;AAAA,QACA,MAAM,kBAAkB,MAAM,eAAe,GAAG;AAAA,QAChD,MAAM;AAAA,IAGV;AAAA,IAEA,MAAM;AAAA,YACN;AAAA,IACA,MAAM,WAAW,MAAM;AAAA;AAAA;AAI3B,eAAe,iBAAiB,CAC9B,SACA,KACe;AAAA,EACf,IAAI;AAAA,IACF,MAAM,UAAU,GAAG;AAAA,IACnB,OAAO,OAAO;AAAA,IAGd,QAAQ,MAAM,+DAA+D,KAAK;AAAA;AAAA;;;AFrJ/E,MAAM,uBAAuB,YAGlC;AAAA,EACiB;AAAA,EACA;AAAA,EAEjB,WAAW,CAAC,MAA0B,UAAiC,CAAC,GAAG;AAAA,IACzE,IAAI,KAAK,YAAY,UAAU,KAAK,EAAE,WAAW,GAAG;AAAA,MAClD,MAAM,IAAI,MAAM,8DAA8D;AAAA,IAChF;AAAA,IAEA,MAAM,sBAAsB,KAAK,QAAQ;AAAA,IACzC,IAAI,CAAC,qBAAqB;AAAA,MACxB,MAAM,IAAI,MAAM,6EAA6E;AAAA,IAC/F;AAAA,IAEA,MAAM;AAAA,MACJ,WAAW,KAAK;AAAA,MAChB,OAAO,KAAK,OAAO;AAAA,MACnB,cAAc;AAAA,MACd,UAAU,mBAAmB,KAAK;AAAA,MAClC,YAAY,QAAQ;AAAA,IACtB,CAAC;AAAA,IAED,KAAK,UAAU,qBAAqB,MAAM,mBAAmB;AAAA,IAC7D,KAAK,OAAO;AAAA;AAAA,OAGE,QAAO,CACrB,SACA,QACe;AAAA,IACf,QAAQ,QAAQ;AAAA,IAChB,MAAM,eAAe,KAAK,KAAK,QAAQ;AAAA,IACvC,IAAI,CAAC,cAAc;AAAA,MACjB,MAAM,IAAI,MAAM,6EAA6E;AAAA,IAC/F;AAAA,IACA,MAAM,MAAM,MAAM,aAAa,QAAQ,EAAE,WAAW,KAAK,KAAK,IAAI,IAAI,IAAI,QAAQ,MAAM,CAAC;AAAA,IACzF,IAAI,CAAC,KAAK;AAAA,MACR,MAAM,IAAI,MAAM,sCAAsC,IAAI,QAAQ,uBAAuB;AAAA,IAC3F;AAAA,IAEA,MAAM,mBAAmB,MAAM,KAAK,KAAK,QAAQ,WAAW,QAAQ;AAAA,MAClE,WAAW,KAAK,KAAK;AAAA,MACrB,IAAI,IAAI;AAAA,IACV,CAAC;AAAA,IACD,IAAI,CAAC,kBAAkB;AAAA,MACrB,MAAM,IAAI,MACR,sCAAsC,IAAI,qCAAqC,IAAI,eACrF;AAAA,IACF;AAAA,IAEA,MAAM,cAA2B;AAAA,MAC/B,IAAI,IAAI;AAAA,MACR,YAAY,IAAI;AAAA,IAClB;AAAA,IAGA,8BAA8B,KAAK,MAAM;AAAA,MACvC,WAAW;AAAA,MACX,WAAW,EAAE,MAAM,YAAY,IAAI,IAAI,YAAY,OAAO,IAAI,GAAG;AAAA,IACnE,CAAC;AAAA,IAED,IAAI;AAAA,MACF,MAAM,eAAe;AAAA,QACnB,SAAS,KAAK;AAAA,QACd;AAAA,QACA;AAAA,QACA,cAAc,CAAC,QACb,uBAAuB,KAAK,KAAK,QAAQ,KAAK,iBAAiB,aAAa;AAAA,QAC9E,eAAe,CAAC,QACd,wBAAwB,KAAK,KAAK,QAAQ,KAAK,iBAAiB,aAAa;AAAA,QAC/E,aAAa,CAAC,OAAO,KAAK,YAAY;AAAA,UACpC,iBAAiB,KAAK,MAAM,OAAO;AAAA,YACjC,WAAW,KAAK,KAAK;AAAA,YACrB,SAAS,IAAI;AAAA,YACb,SAAS;AAAA,YACT,KAAK;AAAA,cACH,OAAO,IAAI;AAAA,cACX,YAAY,IAAI;AAAA,YAClB;AAAA,YACA;AAAA,UACF,CAAC;AAAA;AAAA,QAEH,eAAe,CAAC,MAAM,YACpB,2BACE,KAAK,KAAK,QACV,MACA,SACA,iBAAiB,aACnB;AAAA,QACF,gBAAgB,CAAC,MAAM,YACrB,4BACE,KAAK,KAAK,QACV,MACA,SACA,iBAAiB,aACnB;AAAA,QACF,iBAAiB,CAAC,SAChB,4BACE,KAAK,KAAK,QACV,aACA,MACA,iBAAiB,aACnB;AAAA,MACJ,CAAC;AAAA,MACD,OAAO,OAAO;AAAA,MACd,IAAI,iBAAiB;AAAA,QAAgC;AAAA,MACrD,MAAM;AAAA;AAAA;AAAA,OAIe,iBAAgB,CACvC,SACA,QACkF;AAAA,IAGlF,MAAM,MAAM,MAAM,KAAK,KAAK,QAAQ,cAAc,QAAQ;AAAA,MACxD,WAAW,KAAK,KAAK;AAAA,MACrB,IAAI,QAAQ,IAAI,QAAQ;AAAA,IAC1B,CAAC;AAAA,IACD,OAAO,KAAK,WAAW,YAAY,IAAI,QACnC,EAAE,MAAM,QAAQ,SAAS,IAAI,MAAM,IACnC,EAAE,MAAM,OAAO;AAAA;AAAA,OAGI,aAAY,CACnC,SACA,QACkF;AAAA,IAClF,MAAM,MAAM,MAAM,KAAK,KAAK,QAAQ,cAAc,QAAQ;AAAA,MACxD,WAAW,KAAK,KAAK;AAAA,MACrB,IAAI,QAAQ,IAAI,QAAQ;AAAA,IAC1B,CAAC;AAAA,IACD,IAAI,CAAC;AAAA,MAAK,OAAO,EAAE,MAAM,OAAO;AAAA,IAEhC,IAAI,CAAE,MAAM,iBAAiB,KAAK,SAAS,IAAI,EAAE,GAAI;AAAA,MACnD,OAAO,EAAE,MAAM,QAAQ;AAAA,IACzB;AAAA,IAGA,OAAO,IAAI,WAAW,eAAe,IAAI,QACrC,EAAE,MAAM,QAAQ,SAAS,IAAI,MAAM,IACnC,EAAE,MAAM,OAAO;AAAA;AAEvB;AAEA,SAAS,oBAAoB,CAC3B,MACA,qBACuB;AAAA,EACvB,OAAO;AAAA,IACL,IAAI,KAAK;AAAA,IACT;AAAA,IACA,aAAa,KAAK;AAAA,IAClB,SAAS,KAAK;AAAA,IACd,WAAW,KAAK,YAAY;AAAA,IAC5B,UAAU,KAAK,YAAY;AAAA,EAC7B;AAAA;AAGF,eAAe,gBAAgB,CAC7B,SACA,eACkB;AAAA,EAClB,MAAM,SAAS,MAAM,QAAQ,oBAAoB,UAAU;AAAA,IACzD,WAAW,QAAQ;AAAA,IACnB;AAAA,IACA,UAAU,CAAC,WAAW;AAAA,IACtB,OAAO;AAAA,EACT,CAAC;AAAA,EAED,OAAO,OAAO,MAAM,KAAK,CAAC,SAAS,QAAQ,KAAK,MAAM,CAAC;AAAA;",
15
- "debugId": "71A662855CE2F2BB64756E2164756E21",
14
+ "mappings": ";;AAAA;AACA;AAEA;AAEA,uCAAS;;;ACGT,IAAM,SAAS;AAEf,eAAsB,sBAAsB,CAC1C,QACA,KACA,eACe;AAAA,EACf,IAAI,CAAC,IAAI,WAAW;AAAA,IAClB,MAAM,IAAI,MAAM,sCAAsC,IAAI,kCAAkC;AAAA,EAC9F;AAAA,EACA,MAAM,QAAQ,KACZ;AAAA,IACE,QAAQ;AAAA,MACN;AAAA,QACE,MAAM;AAAA,QACN,SAAS;AAAA,UACP,YAAY,IAAI;AAAA,UAChB,OAAO,IAAI;AAAA,UACX,WAAW,IAAI,UAAU,YAAY;AAAA,QACvC;AAAA,MACF;AAAA,IACF;AAAA,IACA;AAAA,EACF,GACA,EAAE,QAAQ,OAAO,CACnB;AAAA;AAGF,eAAsB,0BAA0B,CAC9C,QACA,MAIA,SACA,eACe;AAAA,EACf,MAAM,QAAQ,KACZ;AAAA,IACE,QAAQ;AAAA,MACN;AAAA,QACE,MAAM;AAAA,QACN,SAAS;AAAA,UACP,YAAY,KAAK;AAAA,UACjB,OAAO,KAAK;AAAA,UACZ,WAAW,KAAK;AAAA,UAChB,QAAQ,KAAK;AAAA,UACb,WAAW,QAAQ;AAAA,UACnB,YAAY,QAAQ;AAAA,UACpB,WAAW,KAAK;AAAA,UAChB,WAAW,KAAK,UAAU,YAAY;AAAA,QACxC;AAAA,MACF;AAAA,IACF;AAAA,IACA;AAAA,EACF,GACA,EAAE,QAAQ,OAAO,CACnB;AAAA;AAGF,eAAsB,2BAA2B,CAC/C,QACA,MAaA,SACA,eACe;AAAA,EACf,MAAM,QAAQ,KACZ;AAAA,IACE,QAAQ;AAAA,MACN;AAAA,QACE,MAAM;AAAA,QACN,SAAS;AAAA,UACP,YAAY,KAAK;AAAA,UACjB,OAAO,KAAK;AAAA,UACZ,WAAW,KAAK;AAAA,UAChB,QAAQ,KAAK;AAAA,UACb,WAAW,QAAQ;AAAA,UACnB,YAAY,QAAQ;AAAA,UACpB,WAAW,KAAK;AAAA,UAChB,QAAQ,sBAAsB,KAAK,QAAQ,sBAAsB,KAAK,KAAK;AAAA,UAC3E,YAAY,kBAAkB,KAAK,IAAI,KAAK,UAAU,EAAE,YAAY;AAAA,aAChE,KAAK,SAAS,EAAE,WAAW,KAAK,OAAO,UAAU,IAAI,CAAC;AAAA,aACtD,KAAK,gBAAgB,YAAY,EAAE,aAAa,KAAK,YAAY,IAAI,CAAC;AAAA,aACtE,KAAK,QAAQ,EAAE,OAAO,KAAK,MAAM,IAAI,CAAC;AAAA,QAC5C;AAAA,MACF;AAAA,IACF;AAAA,IACA;AAAA,EACF,GACA,EAAE,QAAQ,OAAO,CACnB;AAAA;AAGF,eAAsB,2BAA2B,CAC/C,QACA,KACA,MACA,eACe;AAAA,EACf,IAAI,CAAC,KAAK;AAAA,IAAgB;AAAA,EAG1B,MAAM,QAAQ,KACZ;AAAA,IACE,QAAQ;AAAA,MACN;AAAA,QACE,MAAM;AAAA,QACN,SAAS;AAAA,UACP,WAAW,KAAK,QAAQ;AAAA,UACxB,WAAW,KAAK,QAAQ;AAAA,UACxB,WAAW,KAAK,QAAQ,UAAU,YAAY;AAAA,UAC9C,UAAU;AAAA,YACR,MAAM;AAAA,YACN,IAAI,IAAI;AAAA,YACR,OAAO,IAAI;AAAA,YACX,QAAQ,KAAK,IAAI;AAAA,UACnB;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,IACA;AAAA,EACF,GACA,EAAE,QAAQ,OAAO,CACnB;AAAA;AAGF,eAAsB,uBAAuB,CAC3C,QACA,KACA,eACe;AAAA,EACf,MAAM,QAAQ,KACZ;AAAA,IACE,QAAQ,CAAC,EAAE,MAAM,yBAAyB,SAAS,gCAAgC,GAAG,EAAE,CAAC;AAAA,IACzF;AAAA,EACF,GACA,EAAE,QAAQ,OAAO,CACnB;AAAA;AAGF,SAAS,+BAA+B,CACtC,KACA;AAAA,EACA,OAAO;AAAA,IACL,YAAY,IAAI;AAAA,IAChB,OAAO,IAAI;AAAA,IACX,QAAQ,sBAAsB,IAAI,QAAQ,iBAAiB,IAAI,KAAK;AAAA,OAChE,IAAI,SACJ;AAAA,MACE,WAAW,IAAI,OAAO;AAAA,MACtB,WAAW,IAAI,OAAO;AAAA,IACxB,IACA,CAAC;AAAA,OACD,IAAI,QAAQ,EAAE,OAAO,IAAI,MAAM,IAAI,CAAC;AAAA,EAC1C;AAAA;AAGF,SAAS,iBAAiB,CAAC,WAAmB,YAAoC;AAAA,EAChF,IAAI,YAAY;AAAA,IACd,OAAO;AAAA,EACT;AAAA,EAEA,MAAM,IAAI,MAAM,2CAA2C,+BAA+B;AAAA;AAG5F,SAAS,qBAAqB,CAC5B,QACA,SACkD;AAAA,EAClD,IAAI,WAAW,YAAY,WAAW,WAAW;AAAA,IAC/C,MAAM,IAAI,MAAM,wBAAwB,2BAA2B;AAAA,EACrE;AAAA,EAEA,OAAO;AAAA;;;ACjMT,+BAAS;AACT;AAEA,uCAAS;;;ACFT;AAKO,SAAS,cAAc,CAAC,QAA2B;AAAA,EACxD,IAAI,OAAO,SAAS;AAAA,IAClB,MAAM,iBAAiB;AAAA,EACzB;AAAA;AAGK,SAAS,gBAAgB,CAC9B,QACA,OACoD;AAAA,EACpD,OAAO,OAAO,WAAW,aAAa,KAAK,IAAI,cAAc;AAAA;AAGxD,SAAS,kBAAiB,CAAC,OAIzB;AAAA,EACP,IAAI,MAAM,YAAY;AAAA,IACpB,OAAO,MAAM;AAAA,EACf;AAAA,EAEA,MAAM,gBACJ,uBACA,sCAAsC,MAAM,mDAC5C,EAAE,SAAS,EAAE,YAAY,MAAM,YAAY,OAAO,MAAM,MAAM,EAAE,CAClE;AAAA;AAGK,SAAS,0BAA0B,CAAC,SAOjC;AAAA,EAER,OAAO,gBACL,uBACA,kCAAkC,QAAQ,qBAAqB,QAAQ,sCAAsC,QAAQ,QAAQ,gDAAgD,QAAQ,sGACrL;AAAA,IACE,OAAO,QAAQ;AAAA,IACf,SAAS;AAAA,MACP,YAAY,QAAQ;AAAA,MACpB,eAAe,QAAQ;AAAA,MACvB,QAAQ,QAAQ;AAAA,MAChB,WAAW,QAAQ;AAAA,MACnB,WAAW,QAAQ,QAAQ;AAAA,MAC3B,WAAW,QAAQ,QAAQ;AAAA,IAC7B;AAAA,EACF,CACF;AAAA;AAGK,SAAS,yBAAyB,CAAC,SAMvC;AAAA,EACD,OAAO,gBACL,wBACA,sBAAsB,QAAQ,OAAO,iCAAiC,GACtE;AAAA,IACE,OAAO,QAAQ;AAAA,IACf,SAAS;AAAA,MACP,YAAY,QAAQ;AAAA,MACpB,eAAe,QAAQ;AAAA,MACvB,QAAQ,QAAQ;AAAA,SACZ,QAAQ,YAAY,EAAE,WAAW,QAAQ,UAAU,IAAI,CAAC;AAAA,IAC9D;AAAA,EACF,CACF;AAAA;AAGK,SAAS,yBAAyB,CAAC,OAAyB;AAAA,EACjE,OAAO,YAAY,KAAK,KAAK,MAAM,SAAS,0BAA0B,MAAM,UAAU,YAClF,MAAM,QACN;AAAA;AAGC,SAAS,8BAA8B,CAAC,SAKrC;AAAA,EACR,OAAO,gBACL,uBACA,kCAAkC,QAAQ,gDAAgD,QAAQ,QAAQ,oDAAoD,QAAQ,sGACtK;AAAA,IACE,OAAO,QAAQ;AAAA,IACf,SAAS;AAAA,MACP,YAAY,QAAQ;AAAA,MACpB,OAAO,QAAQ;AAAA,MACf,WAAW,QAAQ,QAAQ;AAAA,MAC3B,WAAW,QAAQ,QAAQ;AAAA,IAC7B;AAAA,EACF,CACF;AAAA;AAGK,SAAS,eAAe,CAC7B,UACA,KACoB;AAAA,EACpB,IAAI,CAAC,UAAU;AAAA,IACb,MAAM,gBACJ,uBACA,0CAA0C,IAAI,gBAC9C,EAAE,SAAS,EAAE,YAAY,IAAI,YAAY,OAAO,IAAI,GAAG,EAAE,CAC3D;AAAA,EACF;AAAA,EAEA,OAAO;AAAA;AAGF,SAAS,wBAAwB,CAAC,SAQnB;AAAA,EACpB,IAAI,QAAQ,SAAS;AAAA,IACnB,OAAO,QAAQ;AAAA,EACjB;AAAA,EAEA,MAAM,YAAY,QAAQ,SAAS,UAAU,UAAU,QAAQ,QAAQ,eAAe;AAAA,EAEtF,MAAM,gBACJ,uBACA,kCAAkC,QAAQ,qBAAqB,QAAQ,WAAW,yCAAyC,QAAQ,eACnI;AAAA,IACE,SAAS;AAAA,MACP,YAAY,QAAQ;AAAA,MACpB,eAAe,QAAQ;AAAA,MACvB,QAAQ,QAAQ;AAAA,MAChB,WAAW,QAAQ;AAAA,IACrB;AAAA,EACF,CACF;AAAA;AAGF,SAAS,gBAAgB,GAAU;AAAA,EACjC,MAAM,QAAQ,IAAI,MAAM,yBAAyB;AAAA,EACjD,MAAM,OAAO;AAAA,EACb,OAAO;AAAA;AAGT,SAAS,YAAY,CAAC,OAAyB;AAAA,EAC7C,OAAO,iBAAiB,SAAS,MAAM,SAAS;AAAA;;;AChKlD;AACA;AAEA;;;ACGA,4BAAS;AACT;AACA;;;ACFA,4BAAS;AACT;AAsBA,eAAsB,iBAAiB,CAAC,OAKE;AAAA,EACxC,QAAQ,SAAS,UAAU,MAAM,kBAAkB;AAAA,EACnD,MAAM,WAAgC,CAAC;AAAA,EAEvC,YAAY,MAAM,oBAAoB,OAAO,QAAQ,KAAK,MAAM,GAAG;AAAA,IACjE,MAAM,UAAU,yBAAyB;AAAA,MACvC,SAAS,QAAQ,SAAS,QAAQ,gBAAgB,EAAE;AAAA,MACpD,YAAY,SAAS;AAAA,MACrB;AAAA,MACA,QAAQ,KAAK;AAAA,MACb,MAAM;AAAA,MACN;AAAA,MACA,WAAW,gBAAgB;AAAA,IAC7B,CAAC;AAAA,IAED,MAAM,UAAU,MAAM,QAAQ,YAAY,iBAAiB,QAAQ,EAAE;AAAA,IACrE,IAAI,CAAC,SAAS;AAAA,MACZ,MAAM,iBACJ,uBACA,kCAAkC,SAAS,aAAa,KAAK,cAAc,kBAAkB,QAAQ,iCACrG;AAAA,QACE,SAAS;AAAA,UACP,YAAY,SAAS;AAAA,UACrB;AAAA,UACA,QAAQ,KAAK;AAAA,UACb,WAAW,QAAQ;AAAA,QACrB;AAAA,MACF,CACF;AAAA,IACF;AAAA,IAEA,SAAS,KAAK;AAAA,MACZ;AAAA,MACA;AAAA,SACI,cAAc,eAAe,IAC7B,EAAE,MAAM,EAAE,SAAS,iBAAiB,SAAS,iBAAiB,eAAe,EAAE,EAAE,IACjF,CAAC;AAAA,MACL;AAAA,MACA,KAAK;AAAA,QACH,WAAW,QAAQ;AAAA,QACnB,WAAW,QAAQ;AAAA,MACrB;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,OAAO;AAAA;AAGF,SAAS,gBAAgB,CAAC,OAMI;AAAA,EACnC,QAAQ,aAAa,WAAW;AAAA,EAChC,MAAM,SAA4C,CAAC;AAAA,EAEnD,WAAW,YAAY,MAAM,gBAAgB;AAAA,IAC3C,MAAM,UAAU,kCAAkC,MAAM,SAAS,aAAa,MAAM,KAAK,cAAc,SAAS;AAAA,IAChH,QAAQ,SAAS;AAAA,IAEjB,OAAO,SAAS,QAAQ;AAAA,MACtB,SAAS,MAAM,WAAW,SAAS;AAAA,MACnC,SAAS,SAAS;AAAA,MAClB,QAAQ,CAAC,YAAmE,CAAC,GAAG;AAAA,QAE9E,OAAO,YAAY,SAAS;AAAA,aACvB;AAAA,aACC,QAAQ,EAAE,SAAS,kBAAkB,SAAS,KAAK,SAAS,UAAU,OAAO,EAAE;AAAA,UACnF,QAAQ,UAAU,SAAS,YAAY,IAAI,CAAC,QAAQ,UAAU,MAAM,CAAC,IAAI;AAAA,UACzE,WAAW,SAAS,QAAQ;AAAA,UAC5B,WAAW,SAAS,QAAQ;AAAA,QAC9B,CAAC;AAAA;AAAA,IAEL;AAAA,EACF;AAAA,EAEA,OAAO;AAAA;AAOT,SAAS,iBAAiB,CACxB,SACA,eACA,WACmB;AAAA,EAEnB,IAAI,cAAc,aAAa,UAAU,WAAW;AAAA,IAAG,OAAO;AAAA,EAE9D,MAAM,WAAW,UAAU,KAAK,CAAC,WAAW,CAAC,cAAc,SAAS,MAAM,CAAC;AAAA,EAC3E,IAAI,aAAa,WAAW;AAAA,IAC1B,MAAM,IAAI,MACR,GAAG,yBAAyB,8CAA8C,mCAC5E;AAAA,EACF;AAAA,EAEA,OAAO;AAAA;;;ADzHT,eAAsB,cAAc,CAAC,OAclC;AAAA,EACD,QAAQ,SAAS,UAAU,MAAM,KAAK,QAAQ,eAAe,mBAAmB;AAAA,EAEhF,IAAI,KAAK,SAAS,SAAS,OAAO;AAAA,IAChC,MAAM,iBACJ,uBACA,kCAAkC,SAAS,aAAa,KAAK,4EAC7D;AAAA,MACE,SAAS;AAAA,QACP,YAAY,SAAS;AAAA,QACrB,eAAe,IAAI;AAAA,QACnB,QAAQ,KAAK;AAAA,QACb,WAAW,cAAc;AAAA,MAC3B;AAAA,IACF,CACF;AAAA,EACF;AAAA,EAEA,MAAM,SAAS,MAAM,WAAW,YAAY,EAAE,QAAQ,KAAK,GAAG,CAAC;AAAA,EAC/D,IAAI,cAAc;AAAA,EAClB,MAAM,QAAQ,MAAM,QAAQ,YAAY,WAAW;AAAA,IACjD,SAAS;AAAA,IACT,MAAM,KAAK;AAAA,IACX,QAAQ,eAAe,IAAI,CAAC,aAAa,SAAS,GAAG;AAAA,IACrD,UAAU,EAAE,MAAM,YAAY,IAAI,SAAS,IAAI,OAAO,IAAI,IAAI,QAAQ,KAAK,GAAG;AAAA,IAG9E,SAAS,mBACP,eAAe,MAAM,IAAI,IAAI,CAAC,CAAC,cAAc,IAAI,aAAa,CAAC,CAAC,CAAC,GACjE,eAAe,IAAI,CAAC,aAAa,SAAS,OAAO,CACnD;AAAA,EACF,CAAC;AAAA,EAED,IAAI;AAAA,IACF,MAAM,UAAkC;AAAA,MACtC,WAAW,QAAQ;AAAA,MACnB,YAAY,SAAS;AAAA,MACrB,QAAQ,KAAK;AAAA,MACb,OAAO,IAAI;AAAA,MACX;AAAA,MACA;AAAA,MACA,QAAQ,iBAAiB;AAAA,QACvB,aAAa,QAAQ;AAAA,QACrB;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF,CAAC;AAAA,MACD,QAAQ;AAAA,aACA,UAAS,CAAC,MAAM;AAAA,UACpB,MAAM,MAAM,UACV,iBAAiB,MAAM,QAAQ,MAAM;AAAA,YACnC,eAAe;AAAA,WAChB,CACH;AAAA;AAAA,MAEJ;AAAA,IACF;AAAA,IAEA,MAAM,KAAK,SAAS,QAAQ,OAAO;AAAA,IACnC,eAAe,MAAM;AAAA,IAErB,MAAM,SAAS,MAAM,MAAM,OAAO;AAAA,MAChC,yBAAyB,MAAM;AAAA,MAC/B,eAAe,YAAY,SAAS,WAAW,KAAK,UAAU,IAAI;AAAA,IACpE,CAAC;AAAA,IACD,QAAQ,YAAY,YAAY;AAAA,IAEhC,OAAO;AAAA,MACL;AAAA,MACA,gBAAgB,YAAY;AAAA,MAC5B;AAAA,IACF;AAAA,IACA,OAAO,OAAO;AAAA,IACd,MAAM,MAAM,MAAM,EAAE,MAAM,MAAM,EAAE;AAAA,IAClC,MAAM;AAAA;AAAA;AAIV,gBAAgB,gBAAgB,CAC9B,MACA,QACA,WAC2B;AAAA,EAC3B,iBAAiB,OAAO,MAAM;AAAA,IAC5B,eAAe,MAAM;AAAA,IACrB,MAAM;AAAA,IACN,UAAU;AAAA,EACZ;AAAA;;;AE7GF,4BAAS;AAUT,SAAS,cAAc,CAAC,aAA6D;AAAA,EACnF,MAAM,MAAM,SAAS,cAAc,YAAY,MAAM;AAAA,EAErD,OACE,OAAO,QAAQ,YAAY,QAAQ,QAAQ,aAAa,OAAO,OAAO,IAAI,YAAY;AAAA;AAI1F,SAAS,sBAAsB,CAAC,OAQvB;AAAA,EACP,MAAM,YACJ,MAAM,SAAS,WACX,MAAM,aAAa,iBACnB,MAAM,aAAa;AAAA,EACzB,IAAI;AAAA,IAAW;AAAA,EAEf,MAAM,iBACJ,uBACA,kCAAkC,MAAM,qBAAqB,MAAM,uBACjE,IAAI,MAAM,yBAAyB,MAAM,0CAC3C;AAAA,IACE,SAAS;AAAA,MACP,YAAY,MAAM;AAAA,MAClB,eAAe,MAAM;AAAA,MACrB,QAAQ,MAAM;AAAA,MACd,WAAW,MAAM;AAAA,IACnB;AAAA,EACF,CACF;AAAA;AAGF,eAAsB,cAAc,CAAC,OAalC;AAAA,EACD,QAAQ,SAAS,UAAU,MAAM,KAAK,QAAQ,eAAe,mBAAmB;AAAA,EAEhF,IAAI,KAAK,SAAS,SAAS,OAAO;AAAA,IAChC,MAAM,iBACJ,uBACA,kCAAkC,SAAS,aAAa,KAAK,4EAC7D;AAAA,MACE,SAAS;AAAA,QACP,YAAY,SAAS;AAAA,QACrB,eAAe,IAAI;AAAA,QACnB,QAAQ,KAAK;AAAA,QACb,WAAW,cAAc;AAAA,MAC3B;AAAA,IACF,CACF;AAAA,EACF;AAAA,EAEA,IAAI,CAAC,eAAe,QAAQ,WAAW,GAAG;AAAA,IACxC,MAAM,iBACJ,uBACA,kCAAkC,SAAS,aAAa,KAAK,uGAC7D;AAAA,MACE,SAAS;AAAA,QACP,YAAY,SAAS;AAAA,QACrB,eAAe,IAAI;AAAA,QACnB,QAAQ,KAAK;AAAA,QACb,WAAW,cAAc;AAAA,MAC3B;AAAA,IACF,CACF;AAAA,EACF;AAAA,EAMA,uBAAuB;AAAA,IACrB,cAAc,QAAQ,YAAY,IAAI;AAAA,IACtC,SAAS,QAAQ,YAAY,IAAI;AAAA,IACjC,MAAM,KAAK;AAAA,IACX,YAAY,SAAS;AAAA,IACrB,eAAe,IAAI;AAAA,IACnB,QAAQ,KAAK;AAAA,IACb,WAAW,cAAc;AAAA,EAC3B,CAAC;AAAA,EAED,eAAe,MAAM;AAAA,EAErB,MAAM,SAAS,MAAM,QAAQ,YAAY,IAAI,QAAQ;AAAA,IACnD,SAAS,OAAO,YACd,eAAe,IAAI,CAAC,UAAU;AAAA,MAC5B,MAAM;AAAA,MACN;AAAA,QACE,SAAS,MAAM;AAAA,QACf,WAAW,MAAM,QAAQ;AAAA,WACrB,MAAM,QAAQ,EAAE,SAAS,MAAM,KAAK,QAAQ;AAAA,MAClD;AAAA,IACF,CAAC,CACH;AAAA,IACA,KAAK,KAAK,SAAS;AAAA,IACnB,QAAQ;AAAA,IACR,yBAAyB,MAAM;AAAA,IAC/B,MAAM,KAAK;AAAA,IACX,UAAU;AAAA,MACR,MAAM;AAAA,MACN,IAAI,SAAS;AAAA,MACb,OAAO,IAAI;AAAA,MACX,QAAQ,KAAK;AAAA,IACf;AAAA,IACA,eAAe,YAAY,SAAS,WAAW,KAAK,UAAU,IAAI;AAAA,EACpE,CAAC;AAAA,EAED,eAAe,MAAM;AAAA,EACrB,QAAQ,YAAY,YAAY;AAAA,EAEhC,OAAO;AAAA,IACL;AAAA,IACA,gBAAgB,YAAY;AAAA,IAC5B,aAAa,QAAQ;AAAA,EACvB;AAAA;;;AHtHF,eAAsB,OAAO,CAAC,OAUK;AAAA,EACjC,QAAQ,SAAS,UAAU,MAAM,WAAW,KAAK,WAAW;AAAA,EAC5D,MAAM,mBAAmB;AAAA,IACvB;AAAA,IACA,YAAY,SAAS,MAAM,MAAM;AAAA,EACnC;AAAA,EAEA,eAAe,MAAM;AAAA,EACrB,IAAI;AAAA,EACJ,IAAI;AAAA,EACJ,IAAI;AAAA,EACJ,IAAI;AAAA,IACF,gBAAgB,sBACd,yBAAyB;AAAA,MACvB,SAAS,QAAQ,SAAS,QAAQ,KAAK,OAAO,EAAE;AAAA,MAChD,YAAY,SAAS;AAAA,MACrB,eAAe,IAAI;AAAA,MACnB,QAAQ,KAAK;AAAA,MACb,MAAM;AAAA,MACN,WAAW,KAAK,OAAO;AAAA,IACzB,CAAC,CACH;AAAA,IAEA,MAAM,QAAQ,YAAY,cAAc,aAAa;AAAA,IAGrD,2BACG,MAAM,QAAQ,YAAY,iBAAiB,cAAc,EAAE,IAAI,aAAa;AAAA,IAC/E,iBAAiB,MAAM,kBAAkB;AAAA,MACvC;AAAA,MACA;AAAA,MACA;AAAA,MACA,eAAe,IAAI;AAAA,IACrB,CAAC;AAAA,IACD,OAAO,OAAO;AAAA,IACd,IAAI,iBAAiB,QAAQ,KAAK,MAAM,UAAU;AAAA,MAChD,MAAM,0BAA0B;AAAA,QAC9B,YAAY,SAAS;AAAA,QACrB,eAAe,IAAI;AAAA,QACnB,QAAQ,KAAK;AAAA,QACb,OAAO;AAAA,MACT,CAAC;AAAA,IACH;AAAA,IACA,MAAM;AAAA;AAAA,EAER,MAAM,YAAY,eAAe,IAAI,CAAC,aAAa,SAAS,GAAG;AAAA,EAE/D,MAAM,YAAY,GAAG,IAAI,WAAW,YAAY,KAAK,KAAK;AAAA,EAC1D,MAAM,iBAAiB,MAAM,QAAQ,oBAAoB,UAAU;AAAA,IACjE,WAAW,QAAQ;AAAA,IACnB,IAAI;AAAA,IACJ,eAAe,IAAI;AAAA,IACnB,YAAY,SAAS;AAAA,IACrB,QAAQ,KAAK;AAAA,IACb,WAAW,cAAc;AAAA,IACzB,MAAM,KAAK;AAAA,IACX,QAAQ;AAAA,EACV,CAAC;AAAA,EACD,MAAM,MAAM,gBAAgB,gBAAgB,gBAAgB;AAAA,EAE5D,IAAI;AAAA,EACJ,IAAI;AAAA,EAEJ,IAAI;AAAA,IACF,MAAM,YAAY,MAAM,YAAY;AAAA,MAClC;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,YAAY,MAAM;AAAA,MAClB;AAAA,MACA;AAAA,MACA;AAAA,IACF,CAAC;AAAA,IACD,mBAAmB,UAAU;AAAA,IAC7B,cAAc,UAAU;AAAA,IAExB,IAAI;AAAA,IACJ,IAAI;AAAA,MACF,UAAU,MAAM,QAAQ,oBAAoB,WAAW;AAAA,QACrD,WAAW,QAAQ;AAAA,QACnB,IAAI;AAAA,QACJ,QAAQ;AAAA,QACR,QAAQ;AAAA,UACN,WAAW,iBAAiB;AAAA,UAC5B,WAAW,iBAAiB;AAAA,QAC9B;AAAA,QACA;AAAA,MACF,CAAC;AAAA,MACD,OAAO,OAAO;AAAA,MACd,MAAM,2BAA2B;AAAA,QAC/B,YAAY,SAAS;AAAA,QACrB,eAAe,IAAI;AAAA,QACnB,QAAQ,KAAK;AAAA,QACb;AAAA,QACA,SAAS;AAAA,QACT,OAAO;AAAA,MACT,CAAC;AAAA;AAAA,IAEH,MAAM,MAAM,iBAAiB,SAAS,gBAAgB;AAAA,IAEtD,OAAO;AAAA,MACL,KAAK;AAAA,MACL,SAAS;AAAA,MACT,gBAAgB,UAAU;AAAA,IAC5B;AAAA,IACA,OAAO,OAAO;AAAA,IACd,IAAI,CAAC,kBAAkB;AAAA,MACrB,MAAM,SAAS,iBAAiB,QAAQ,KAAK;AAAA,MAC7C,MAAM,eACJ,WAAW,WACP,0BAA0B;AAAA,QACxB,YAAY,SAAS;AAAA,QACrB,eAAe,IAAI;AAAA,QACnB,QAAQ,KAAK;AAAA,QACb;AAAA,QACA,OAAO;AAAA,MACT,CAAC,IACD;AAAA,MACN,MAAM,gBAAgB,MAAM,QAAQ,oBACjC,WAAW;AAAA,QACV,WAAW,QAAQ;AAAA,QACnB,IAAI;AAAA,QACJ;AAAA,QACA;AAAA,QACA,OAAO,mBAAmB,cAAc;AAAA,UACtC,cAAc;AAAA,UACd,aAAa,WAAW,cAAc,sBAAsB;AAAA,UAC5D,SAAS;AAAA,YACP,YAAY,SAAS;AAAA,YACrB,eAAe,IAAI;AAAA,YACnB,QAAQ,KAAK;AAAA,YACb;AAAA,UACF;AAAA,QACF,CAAC;AAAA,MACH,CAAC,EACA,MAAM,MAAM,IAAI;AAAA,MAEnB,IAAI,eAAe;AAAA,QACjB,MAAM,MAAM,iBAAiB,eAAe,gBAAgB;AAAA,MAC9D;AAAA,MAEA,MAAM;AAAA,IACR;AAAA,IAEA,MAAM;AAAA;AAAA;AAIV,eAAe,WAAW,CAAC,OAUM;AAAA,EAC/B,QAAQ,MAAM,KAAK,SAAS;AAAA,SACrB;AAAA,MACH,OAAO,eAAe,KAAK;AAAA,SACxB;AAAA,MACH,OAAO,eAAe,KAAK;AAAA;AAAA;;;AFzL1B,MAAM,uCAAuC,MAAM;AAAA,EAGnC;AAAA,EAFH,OAAO;AAAA,EAEzB,WAAW,CAAU,KAAwB;AAAA,IAC3C,MAAM,sCAAsC,IAAI,0BAA0B;AAAA,IADvD;AAAA;AAGvB;AAEA,eAAsB,cAAc,CAAC,OAAwD;AAAA,EAC3F,QAAQ,SAAS,KAAK,eAAe;AAAA,EACrC,MAAM,MAAM,EAAE,IAAI,WAAW,IAAI,YAAY,WAAW,WAAW;AAAA,EACnE,MAAM,SAAS,MAAM,UAAU,IAAI,gBAAgB,EAAE;AAAA,EACrD,IAAI,WAAW,cAAc,QAAQ,IAAI;AAAA,IACvC,MAAM,IAAI,MACR,sCAAsC,WAAW,2BAA2B,WAAW,oBAAoB,QAAQ,MACrH;AAAA,EACF;AAAA,EACA,MAAM,aAAa,sBAAsB,QAAQ,IAAI,QAAQ,OAAO,EAAE,eAAe;AAAA,IACnF,MAAM;AAAA,IACN,IAAI,IAAI;AAAA,EACV,CAAC;AAAA,EACD,MAAM,QAAiC,CAAC;AAAA,EACxC,IAAI;AAAA,EACJ,IAAI;AAAA,EACJ,IAAI,WAAW;AAAA,EAEf,eAAe,MAAM;AAAA,EAErB,IAAI;AAAA,IACF,IAAI;AAAA,MACF,aAAa,MAAM,QAAQ,oBAAoB,MAAM;AAAA,QACnD,WAAW,QAAQ;AAAA,QACnB,IAAI,IAAI;AAAA,MACV,CAAC;AAAA,MACD,OAAO,OAAO;AAAA,MACd,MAAM,WAAW,MAAM,QAAQ,oBAAoB,QAAQ;AAAA,QACzD,WAAW,QAAQ;AAAA,QACnB,IAAI,IAAI;AAAA,MACV,CAAC;AAAA,MACD,IACE,UAAU,eAAe,WAAW,cACpC,SAAS,gBAAgB,WAAW,aACpC;AAAA,QACA,MAAM,IAAI,+BAA+B,QAAQ;AAAA,MACnD;AAAA,MACA,MAAM;AAAA;AAAA,IAER,MAAM,MAAM,eAAe,UAAU;AAAA,IAErC,MAAM,WAAW,gBAAgB,QAAQ,UAAU,QAAQ,IAAI,UAAU,GAAG,GAAG;AAAA,IAE/E,YAAY,WAAW,SAAS,SAAS,MAAM,MAAM,QAAQ,GAAG;AAAA,MAC9D,MAAM,aAAa,MAAM,QAAQ;AAAA,QAC/B;AAAA,QACA;AAAA,QACA,MAAM,KAAK;AAAA,QACX;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA,eAAe,MAAM;AAAA,QACrB,gBAAgB,MAAM;AAAA,MACxB,CAAC;AAAA,MAED,MAAM,KAAK,UAAU;AAAA,MACrB,eAAe,WAAW;AAAA,MAE1B,MAAM,MAAM,kBAAkB,UAAU;AAAA,IAC1C;AAAA,IAEA,IAAI;AAAA,IACJ,IAAI;AAAA,MACF,MAAM,MAAM,QAAQ,oBAAoB,OAAO;AAAA,QAC7C,WAAW,QAAQ;AAAA,QACnB,IAAI,IAAI;AAAA,QACR,QAAQ;AAAA,QACR,QAAQ,eACJ;AAAA,UACE,WAAW,aAAa;AAAA,UACxB,WAAW,aAAa;AAAA,QAC1B,IACA;AAAA,MACN,CAAC;AAAA,MACD,OAAO,OAAO;AAAA,MACd,IAAI,cAAc;AAAA,QAChB,MAAM,+BAA+B;AAAA,UACnC,YAAY,SAAS;AAAA,UACrB,OAAO,IAAI;AAAA,UACX,SAAS;AAAA,UACT,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AAAA,MACA,MAAM;AAAA;AAAA,IAER,MAAM,cAAc;AAAA,SACf;AAAA,MACH,YAAY,mBAAkB;AAAA,QAC5B,YAAY,SAAS;AAAA,QACrB,OAAO,IAAI;AAAA,QACX,YAAY,IAAI;AAAA,MAClB,CAAC;AAAA,IACH;AAAA,IAEA,WAAW;AAAA,IACX,MAAM,kBAAkB,MAAM,eAAe,WAAW;AAAA,IACxD,OAAO;AAAA,MACL,KAAK;AAAA,MACL;AAAA,MACA,SAAS;AAAA,IACX;AAAA,IACA,OAAO,OAAO;AAAA,IACd,MAAM,gBAAgB,0BAA0B,KAAK;AAAA,IACrD,IAAI,cAAc,CAAC,UAAU;AAAA,MAC3B,MAAM,SAAS,iBAAiB,QAAQ,KAAK;AAAA,MAC7C,IAAI;AAAA,QACF,MAAM,UAAU,oBAAmB,OAAO;AAAA,UACxC,cAAc;AAAA,UACd,aAAa,WAAW,cAAc,sBAAsB;AAAA,UAC5D,SAAS,EAAE,YAAY,WAAW,YAAY,OAAO,IAAI,GAAG;AAAA,QAC9D,CAAC;AAAA,QACD,MAAM,MAAM,MAAM,QAAQ,oBAAoB,OAAO;AAAA,UACnD,WAAW,QAAQ;AAAA,UACnB,IAAI,IAAI;AAAA,UACR;AAAA,UACA,OAAO;AAAA,QACT,CAAC;AAAA,QACD,IAAI,WAAW,YAAY,IAAI,WAAW,UAAU;AAAA,UAClD,MAAM,cAAc,eAAe,KAAK,OAAO;AAAA,QACjD;AAAA,QACA,MAAM,kBAAkB,MAAM,eAAe,GAAG;AAAA,QAChD,MAAM;AAAA,IAGV;AAAA,IAEA,MAAM;AAAA,YACN;AAAA,IACA,MAAM,WAAW,MAAM;AAAA;AAAA;AAI3B,eAAe,iBAAiB,CAC9B,SACA,KACe;AAAA,EACf,IAAI;AAAA,IACF,MAAM,UAAU,GAAG;AAAA,IACnB,OAAO,OAAO;AAAA,IAGd,QAAQ,MAAM,+DAA+D,KAAK;AAAA;AAAA;;;AFrJ/E,MAAM,uBAAuB,YAGlC;AAAA,EACiB;AAAA,EACA;AAAA,EAEjB,WAAW,CAAC,MAA0B,UAAiC,CAAC,GAAG;AAAA,IACzE,IAAI,KAAK,YAAY,UAAU,KAAK,EAAE,WAAW,GAAG;AAAA,MAClD,MAAM,IAAI,MAAM,8DAA8D;AAAA,IAChF;AAAA,IAEA,MAAM,sBAAsB,KAAK,QAAQ;AAAA,IACzC,IAAI,CAAC,qBAAqB;AAAA,MACxB,MAAM,IAAI,MAAM,6EAA6E;AAAA,IAC/F;AAAA,IAEA,MAAM;AAAA,MACJ,WAAW,KAAK;AAAA,MAChB,OAAO,KAAK,OAAO;AAAA,MACnB,cAAc;AAAA,MACd,UAAU,mBAAmB,KAAK;AAAA,MAClC,YAAY,QAAQ;AAAA,IACtB,CAAC;AAAA,IAED,KAAK,UAAU,qBAAqB,MAAM,mBAAmB;AAAA,IAC7D,KAAK,OAAO;AAAA;AAAA,OAGE,QAAO,CACrB,SACA,QACe;AAAA,IACf,QAAQ,QAAQ;AAAA,IAChB,MAAM,eAAe,KAAK,KAAK,QAAQ;AAAA,IACvC,IAAI,CAAC,cAAc;AAAA,MACjB,MAAM,IAAI,MAAM,6EAA6E;AAAA,IAC/F;AAAA,IACA,MAAM,MAAM,MAAM,aAAa,QAAQ,EAAE,WAAW,KAAK,KAAK,IAAI,IAAI,IAAI,QAAQ,MAAM,CAAC;AAAA,IACzF,IAAI,CAAC,KAAK;AAAA,MACR,MAAM,IAAI,MAAM,sCAAsC,IAAI,QAAQ,uBAAuB;AAAA,IAC3F;AAAA,IAEA,MAAM,mBAAmB,MAAM,KAAK,KAAK,QAAQ,WAAW,QAAQ;AAAA,MAClE,WAAW,KAAK,KAAK;AAAA,MACrB,IAAI,IAAI;AAAA,IACV,CAAC;AAAA,IACD,IAAI,CAAC,kBAAkB;AAAA,MACrB,MAAM,IAAI,MACR,sCAAsC,IAAI,qCAAqC,IAAI,eACrF;AAAA,IACF;AAAA,IAEA,MAAM,cAA2B;AAAA,MAC/B,IAAI,IAAI;AAAA,MACR,YAAY,IAAI;AAAA,IAClB;AAAA,IAGA,8BAA8B,KAAK,MAAM;AAAA,MACvC,WAAW;AAAA,MACX,WAAW,EAAE,MAAM,YAAY,IAAI,IAAI,YAAY,OAAO,IAAI,GAAG;AAAA,IACnE,CAAC;AAAA,IAED,IAAI;AAAA,MACF,MAAM,eAAe;AAAA,QACnB,SAAS,KAAK;AAAA,QACd;AAAA,QACA;AAAA,QACA,cAAc,CAAC,QACb,uBAAuB,KAAK,KAAK,QAAQ,KAAK,iBAAiB,aAAa;AAAA,QAC9E,eAAe,CAAC,QACd,wBAAwB,KAAK,KAAK,QAAQ,KAAK,iBAAiB,aAAa;AAAA,QAC/E,aAAa,CAAC,OAAO,KAAK,YAAY;AAAA,UACpC,iBAAiB,KAAK,MAAM,OAAO;AAAA,YACjC,WAAW,KAAK,KAAK;AAAA,YACrB,SAAS,IAAI;AAAA,YACb,SAAS;AAAA,YACT,KAAK;AAAA,cACH,OAAO,IAAI;AAAA,cACX,YAAY,IAAI;AAAA,YAClB;AAAA,YACA;AAAA,UACF,CAAC;AAAA;AAAA,QAEH,eAAe,CAAC,MAAM,YACpB,2BACE,KAAK,KAAK,QACV,MACA,SACA,iBAAiB,aACnB;AAAA,QACF,gBAAgB,CAAC,MAAM,YACrB,4BACE,KAAK,KAAK,QACV,MACA,SACA,iBAAiB,aACnB;AAAA,QACF,iBAAiB,CAAC,SAChB,4BACE,KAAK,KAAK,QACV,aACA,MACA,iBAAiB,aACnB;AAAA,MACJ,CAAC;AAAA,MACD,OAAO,OAAO;AAAA,MACd,IAAI,iBAAiB;AAAA,QAAgC;AAAA,MACrD,MAAM;AAAA;AAAA;AAAA,OAIe,iBAAgB,CACvC,SACA,QACkF;AAAA,IAGlF,MAAM,MAAM,MAAM,KAAK,KAAK,QAAQ,cAAc,QAAQ;AAAA,MACxD,WAAW,KAAK,KAAK;AAAA,MACrB,IAAI,QAAQ,IAAI,QAAQ;AAAA,IAC1B,CAAC;AAAA,IACD,OAAO,KAAK,WAAW,YAAY,IAAI,QACnC,EAAE,MAAM,QAAQ,SAAS,IAAI,MAAM,IACnC,EAAE,MAAM,OAAO;AAAA;AAAA,OAGI,aAAY,CACnC,SACA,QACkF;AAAA,IAClF,MAAM,MAAM,MAAM,KAAK,KAAK,QAAQ,cAAc,QAAQ;AAAA,MACxD,WAAW,KAAK,KAAK;AAAA,MACrB,IAAI,QAAQ,IAAI,QAAQ;AAAA,IAC1B,CAAC;AAAA,IACD,IAAI,CAAC;AAAA,MAAK,OAAO,EAAE,MAAM,OAAO;AAAA,IAEhC,IAAI,CAAE,MAAM,iBAAiB,KAAK,SAAS,IAAI,EAAE,GAAI;AAAA,MACnD,OAAO,EAAE,MAAM,QAAQ;AAAA,IACzB;AAAA,IAGA,OAAO,IAAI,WAAW,eAAe,IAAI,QACrC,EAAE,MAAM,QAAQ,SAAS,IAAI,MAAM,IACnC,EAAE,MAAM,OAAO;AAAA;AAEvB;AAEA,SAAS,oBAAoB,CAC3B,MACA,qBACuB;AAAA,EACvB,OAAO;AAAA,IACL,IAAI,KAAK;AAAA,IACT;AAAA,IACA,aAAa,KAAK;AAAA,IAClB,SAAS,KAAK;AAAA,IACd,WAAW,KAAK,YAAY;AAAA,IAC5B,UAAU,KAAK,YAAY;AAAA,EAC7B;AAAA;AAGF,eAAe,gBAAgB,CAC7B,SACA,eACkB;AAAA,EAClB,MAAM,SAAS,MAAM,QAAQ,oBAAoB,UAAU;AAAA,IACzD,WAAW,QAAQ;AAAA,IACnB;AAAA,IACA,UAAU,CAAC,WAAW;AAAA,IACtB,OAAO;AAAA,EACT,CAAC;AAAA,EAED,OAAO,OAAO,MAAM,KAAK,CAAC,SAAS,QAAQ,KAAK,MAAM,CAAC;AAAA;",
15
+ "debugId": "5FFC283C2E39D35B64756E2164756E21",
16
16
  "names": []
17
17
  }
@@ -3,7 +3,13 @@ import type { DatasetVersion, DatasetVersionRef } from "@sixb/core/lake-storage"
3
3
  import type { PipelineWorkerContext } from "./types";
4
4
  export interface ResolvedStepInput {
5
5
  readonly name: string;
6
+ /** The registered dataset. */
6
7
  readonly dataset: DatasetDefinition;
8
+ /** Present when the step input is `dataset.pick(...)`: the step reads only those columns. */
9
+ readonly pick?: {
10
+ readonly dataset: DatasetDefinition;
11
+ readonly columns: readonly string[];
12
+ };
7
13
  readonly version: DatasetVersion;
8
14
  readonly ref: DatasetVersionRef;
9
15
  }
@@ -13,5 +19,11 @@ export declare function resolveStepInputs(input: {
13
19
  readonly step: PipelineStepDefinition;
14
20
  readonly pipelineRunId: string;
15
21
  }): Promise<readonly ResolvedStepInput[]>;
16
- export declare function createStepInputs(lakeStorage: PipelineWorkerContext["lakeStorage"], resolved: readonly ResolvedStepInput[], signal: AbortSignal): PipelineStepRunContext["inputs"];
22
+ export declare function createStepInputs(input: {
23
+ readonly lakeStorage: PipelineWorkerContext["lakeStorage"];
24
+ readonly pipeline: PipelineDefinition;
25
+ readonly step: PipelineStepDefinition;
26
+ readonly resolvedInputs: readonly ResolvedStepInput[];
27
+ readonly signal: AbortSignal;
28
+ }): PipelineStepRunContext["inputs"];
17
29
  //# sourceMappingURL=step-inputs.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"step-inputs.d.ts","sourceRoot":"","sources":["../src/step-inputs.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,iBAAiB,EACjB,kBAAkB,EAClB,sBAAsB,EAEtB,sBAAsB,EACvB,MAAM,YAAY,CAAA;AAEnB,OAAO,KAAK,EACV,cAAc,EACd,iBAAiB,EAElB,MAAM,yBAAyB,CAAA;AAEhC,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,SAAS,CAAA;AAEpD,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,OAAO,EAAE,iBAAiB,CAAA;IACnC,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAA;IAChC,QAAQ,CAAC,GAAG,EAAE,iBAAiB,CAAA;CAChC;AAED,wBAAsB,iBAAiB,CAAC,KAAK,EAAE;IAC7C,QAAQ,CAAC,OAAO,EAAE,qBAAqB,CAAA;IACvC,QAAQ,CAAC,QAAQ,EAAE,kBAAkB,CAAA;IACrC,QAAQ,CAAC,IAAI,EAAE,sBAAsB,CAAA;IACrC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAA;CAC/B,GAAG,OAAO,CAAC,SAAS,iBAAiB,EAAE,CAAC,CA2CxC;AAED,wBAAgB,gBAAgB,CAC9B,WAAW,EAAE,qBAAqB,CAAC,aAAa,CAAC,EACjD,QAAQ,EAAE,SAAS,iBAAiB,EAAE,EACtC,MAAM,EAAE,WAAW,GAClB,sBAAsB,CAAC,QAAQ,CAAC,CAoBlC"}
1
+ {"version":3,"file":"step-inputs.d.ts","sourceRoot":"","sources":["../src/step-inputs.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,iBAAiB,EACjB,kBAAkB,EAClB,sBAAsB,EAEtB,sBAAsB,EACvB,MAAM,YAAY,CAAA;AAGnB,OAAO,KAAK,EACV,cAAc,EACd,iBAAiB,EAElB,MAAM,yBAAyB,CAAA;AAEhC,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,SAAS,CAAA;AAEpD,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,8BAA8B;IAC9B,QAAQ,CAAC,OAAO,EAAE,iBAAiB,CAAA;IACnC,6FAA6F;IAC7F,QAAQ,CAAC,IAAI,CAAC,EAAE;QACd,QAAQ,CAAC,OAAO,EAAE,iBAAiB,CAAA;QACnC,QAAQ,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,CAAA;KACpC,CAAA;IACD,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAA;IAChC,QAAQ,CAAC,GAAG,EAAE,iBAAiB,CAAA;CAChC;AAED,wBAAsB,iBAAiB,CAAC,KAAK,EAAE;IAC7C,QAAQ,CAAC,OAAO,EAAE,qBAAqB,CAAA;IACvC,QAAQ,CAAC,QAAQ,EAAE,kBAAkB,CAAA;IACrC,QAAQ,CAAC,IAAI,EAAE,sBAAsB,CAAA;IACrC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAA;CAC/B,GAAG,OAAO,CAAC,SAAS,iBAAiB,EAAE,CAAC,CA8CxC;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE;IACtC,QAAQ,CAAC,WAAW,EAAE,qBAAqB,CAAC,aAAa,CAAC,CAAA;IAC1D,QAAQ,CAAC,QAAQ,EAAE,kBAAkB,CAAA;IACrC,QAAQ,CAAC,IAAI,EAAE,sBAAsB,CAAA;IACrC,QAAQ,CAAC,cAAc,EAAE,SAAS,iBAAiB,EAAE,CAAA;IACrD,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAA;CAC7B,GAAG,sBAAsB,CAAC,QAAQ,CAAC,CAyBnC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@sixb/pipeline-worker",
3
- "version": "0.1.20",
3
+ "version": "0.1.22",
4
4
  "description": "Worker that runs Sixb pipelines",
5
5
  "keywords": [
6
6
  "sixb",
@@ -35,10 +35,10 @@
35
35
  "prepack": "bun run build"
36
36
  },
37
37
  "peerDependencies": {
38
- "@sixb/core": "0.1.20"
38
+ "@sixb/core": "0.1.22"
39
39
  },
40
40
  "devDependencies": {
41
- "@sixb/core": "0.1.20",
41
+ "@sixb/core": "0.1.22",
42
42
  "@types/bun": "latest",
43
43
  "typescript": "^5.7.0"
44
44
  },
@@ -6,7 +6,8 @@ import type {
6
6
  PipelineStepRunContext,
7
7
  } from "@sixb/core"
8
8
  import { createSixbError } from "@sixb/core/internal/errors"
9
- import type { DatasetVersion } from "@sixb/core/lake-storage"
9
+ import { runStepLineage } from "@sixb/core/internal/pipelines"
10
+ import { type DatasetVersion, resolveStepLineage } from "@sixb/core/lake-storage"
10
11
  import { throwIfAborted } from "./errors"
11
12
  import { createStepInputs, type ResolvedStepInput } from "./step-inputs"
12
13
  import type { PipelineJob, PipelineLogSession, PipelineWorkerContext } from "./types"
@@ -49,12 +50,13 @@ export async function executeRunStep(input: {
49
50
  dataset: outputDataset,
50
51
  mode: step.mode,
51
52
  inputs: resolvedInputs.map((resolved) => resolved.ref),
52
- producer: {
53
- kind: "pipeline",
54
- id: pipeline.id,
55
- runId: job.id,
56
- stepId: step.id,
57
- },
53
+ producer: { kind: "pipeline", id: pipeline.id, runId: job.id, stepId: step.id },
54
+ // Through every upstream hop: each column the step reads stands for what its input version
55
+ // records. The output is the only registered dataset the step's own lineage reads.
56
+ lineage: resolveStepLineage(
57
+ runStepLineage(step, new Map([[outputDataset.id, outputDataset]])),
58
+ resolvedInputs.map((resolved) => resolved.version)
59
+ ),
58
60
  })
59
61
 
60
62
  try {
@@ -65,7 +67,13 @@ export async function executeRunStep(input: {
65
67
  runId: job.id,
66
68
  signal,
67
69
  logger,
68
- inputs: createStepInputs(runtime.lakeStorage, resolvedInputs, signal),
70
+ inputs: createStepInputs({
71
+ lakeStorage: runtime.lakeStorage,
72
+ pipeline,
73
+ step,
74
+ resolvedInputs,
75
+ signal,
76
+ }),
69
77
  output: {
70
78
  async writeRows(rows) {
71
79
  await write.writeRows(
@@ -122,6 +122,7 @@ export async function executeSqlStep(input: {
122
122
  {
123
123
  dataset: input.dataset,
124
124
  versionId: input.version.versionId,
125
+ ...(input.pick && { columns: input.pick.columns }),
125
126
  },
126
127
  ])
127
128
  ),
@@ -6,6 +6,7 @@ import type {
6
6
  PipelineStepRunContext,
7
7
  } from "@sixb/core"
8
8
  import { createSixbError } from "@sixb/core/internal/errors"
9
+ import { isDatasetPick, stepInputColumns } from "@sixb/core/internal/pipelines"
9
10
  import type {
10
11
  DatasetVersion,
11
12
  DatasetVersionRef,
@@ -16,7 +17,13 @@ import type { PipelineWorkerContext } from "./types"
16
17
 
17
18
  export interface ResolvedStepInput {
18
19
  readonly name: string
20
+ /** The registered dataset. */
19
21
  readonly dataset: DatasetDefinition
22
+ /** Present when the step input is `dataset.pick(...)`: the step reads only those columns. */
23
+ readonly pick?: {
24
+ readonly dataset: DatasetDefinition
25
+ readonly columns: readonly string[]
26
+ }
20
27
  readonly version: DatasetVersion
21
28
  readonly ref: DatasetVersionRef
22
29
  }
@@ -60,6 +67,9 @@ export async function resolveStepInputs(input: {
60
67
  resolved.push({
61
68
  name,
62
69
  dataset,
70
+ ...(isDatasetPick(declaredDataset)
71
+ ? { pick: { dataset: declaredDataset, columns: stepInputColumns(declaredDataset) } }
72
+ : {}),
63
73
  version,
64
74
  ref: {
65
75
  datasetId: version.datasetId,
@@ -71,24 +81,31 @@ export async function resolveStepInputs(input: {
71
81
  return resolved
72
82
  }
73
83
 
74
- export function createStepInputs(
75
- lakeStorage: PipelineWorkerContext["lakeStorage"],
76
- resolved: readonly ResolvedStepInput[],
77
- signal: AbortSignal
78
- ): PipelineStepRunContext["inputs"] {
84
+ export function createStepInputs(input: {
85
+ readonly lakeStorage: PipelineWorkerContext["lakeStorage"]
86
+ readonly pipeline: PipelineDefinition
87
+ readonly step: PipelineStepDefinition
88
+ readonly resolvedInputs: readonly ResolvedStepInput[]
89
+ readonly signal: AbortSignal
90
+ }): PipelineStepRunContext["inputs"] {
91
+ const { lakeStorage, signal } = input
79
92
  const inputs: Record<string, PipelineStepInput> = {}
80
93
 
81
- for (const input of resolved) {
82
- inputs[input.name] = {
83
- dataset: input.dataset,
84
- version: input.version,
94
+ for (const resolved of input.resolvedInputs) {
95
+ const subject = `[SixbPipelineWorker] Pipeline '${input.pipeline.id}' step '${input.step.id}' input '${resolved.name}'`
96
+ const { pick } = resolved
97
+
98
+ inputs[resolved.name] = {
99
+ dataset: pick?.dataset ?? resolved.dataset,
100
+ version: resolved.version,
85
101
  readRows(readInput: Omit<ReadDatasetRowsInput, "datasetId" | "versionId"> = {}) {
86
102
  // Step readers stay pinned even if newer versions commit while the handler runs.
87
103
  return lakeStorage.readRows({
88
104
  ...readInput,
105
+ ...(pick && { columns: readPickedColumns(subject, pick.columns, readInput.columns) }),
89
106
  signal: readInput.signal ? AbortSignal.any([signal, readInput.signal]) : signal,
90
- datasetId: input.dataset.id,
91
- versionId: input.version.versionId,
107
+ datasetId: resolved.dataset.id,
108
+ versionId: resolved.version.versionId,
92
109
  })
93
110
  },
94
111
  }
@@ -96,3 +113,25 @@ export function createStepInputs(
96
113
 
97
114
  return inputs
98
115
  }
116
+
117
+ /**
118
+ * The version records that the step read only the picked columns, which keeps the markings of the
119
+ * others out of its output. Reading another column would make that record false.
120
+ */
121
+ function readPickedColumns(
122
+ subject: string,
123
+ pickedColumns: readonly string[],
124
+ requested: readonly string[] | undefined
125
+ ): readonly string[] {
126
+ // Lake providers read every column for an empty list.
127
+ if (requested === undefined || requested.length === 0) return pickedColumns
128
+
129
+ const unpicked = requested.find((column) => !pickedColumns.includes(column))
130
+ if (unpicked !== undefined) {
131
+ throw new Error(
132
+ `${subject} reads column '${unpicked}', which its pick leaves out. Add '${unpicked}' to the pick to read it.`
133
+ )
134
+ }
135
+
136
+ return requested
137
+ }