@mevdragon/vidfarm-devcli 0.5.2 → 0.6.0

package/dist/src/services/serverless-jobs.js

@@ -0,0 +1,1082 @@
+import { DynamoDBClient } from "@aws-sdk/client-dynamodb";
+import { DynamoDBDocumentClient, GetCommand, PutCommand, QueryCommand, ScanCommand, TransactWriteCommand, UpdateCommand } from "@aws-sdk/lib-dynamodb";
+import { SFNClient, StartExecutionCommand } from "@aws-sdk/client-sfn";
+import { config } from "../config.js";
+import { createId } from "../lib/ids.js";
+import { devLog } from "../lib/dev-log.js";
+import { nowIso } from "../lib/time.js";
+import { ActiveJobLimitExceededError } from "./job-capacity.js";
+import { createLocalDynamoClient } from "./local-dynamo.js";
+// `vidfarm serve` (RECORDS_DRIVER=local) swaps in the disk-backed shim so jobs
+// queue/execute fully locally; deployed environments keep the real client.
+const dynamodb = config.RECORDS_DRIVER === "local"
+    ? createLocalDynamoClient()
+    : DynamoDBDocumentClient.from(new DynamoDBClient({}), {
+        marshallOptions: {
+            removeUndefinedValues: true
+        }
+    });
+const sfn = new SFNClient({});
+const ACTIVE_ROOT_JOB_STATUSES = ["queued", "running", "waiting_for_provider", "waiting_for_child", "waiting_for_human"];
+const ACTIVE_JOB_STATUSES = new Set(ACTIVE_ROOT_JOB_STATUSES);
+export class ServerlessJobsService {
+    async createRootJob(input) {
+        const tableName = requireMainTableName();
+        const createdAt = nowIso();
+        const job = {
+            id: createId("job"),
+            templateId: input.templateId,
+            operationName: input.operationName,
+            workflowName: input.workflowName,
+            tracer: input.tracer,
+            status: "queued",
+            customerId: input.customer.id,
+            payload: {
+                ...input.payload,
+                _providerHint: input.providerHint ?? null
+            },
+            result: null,
+            error: null,
+            progress: 0,
+            webhookUrl: normalizeWebhookUrl(input.webhookUrl) ?? normalizeWebhookUrl(input.customer.defaultWebhookUrl),
+            parentJobId: null,
+            priority: 100,
+            attemptCount: 0,
+            maxAttempts: 6,
+            runAfter: createdAt,
+            createdAt,
+            updatedAt: createdAt,
+            startedAt: null,
+            completedAt: null
+        };
+        await this.persistRootAndStart(tableName, job);
+        devLog("jobs.root.created", {
+            job_id: job.id,
+            customer_id: job.customerId,
+            template_id: job.templateId,
+            operation_name: job.operationName,
+            workflow_name: job.workflowName,
+            tracer: job.tracer
+        });
+        return job;
+    }
+    async createChildJob(input) {
+        const createdAt = nowIso();
+        const job = {
+            id: createId("job"),
+            templateId: input.templateId,
+            operationName: input.operationName,
+            workflowName: input.workflowName,
+            tracer: input.tracer,
+            status: "queued",
+            customerId: input.customerId,
+            payload: {
+                ...input.payload,
+                _providerHint: input.providerHint ?? null
+            },
+            result: null,
+            error: null,
+            progress: 0,
+            webhookUrl: null,
+            parentJobId: input.parentJobId,
+            priority: 110,
+            attemptCount: 0,
+            maxAttempts: 6,
+            runAfter: createdAt,
+            createdAt,
+            updatedAt: createdAt,
+            startedAt: null,
+            completedAt: null
+        };
+        await dynamodb.send(new PutCommand({
+            TableName: requireMainTableName(),
+            Item: {
+                ...jobToItem(job),
+                parent_job_id: input.parentJobId
+            },
+            ConditionExpression: "attribute_not_exists(pk)"
+        }));
+        await dynamodb.send(new PutCommand({
+            TableName: requireMainTableName(),
+            Item: {
+                pk: jobPk(input.parentJobId),
+                sk: `CHILD#${createdAt}#${job.id}`,
+                entity_type: "JobChildPointer",
+                child_job_id: job.id,
+                job_id: input.parentJobId,
+                customer_id: input.customerId,
+                created_at: createdAt
+            }
+        }));
+        try {
+            const executionArn = await this.startWorkflow(job);
+            if (executionArn) {
+                await dynamodb.send(new UpdateCommand({
+                    TableName: requireMainTableName(),
+                    Key: jobKey(job.id),
+                    UpdateExpression: "set step_execution_arn = :arn, updated_at = :updated",
+                    ExpressionAttributeValues: {
+                        ":arn": executionArn,
+                        ":updated": nowIso()
+                    }
+                }));
+            }
+        }
+        catch (error) {
+            devLog("jobs.child.workflow_start_failed", {
+                job_id: job.id,
+                parent_job_id: input.parentJobId,
+                error: error instanceof Error ? error.message : String(error)
+            }, "error");
+        }
+        devLog("jobs.child.created", {
+            job_id: job.id,
+            parent_job_id: input.parentJobId,
+            customer_id: job.customerId,
+            template_id: job.templateId,
+            operation_name: job.operationName,
+            workflow_name: job.workflowName,
+            tracer: job.tracer
+        });
+        return job;
+    }
+    async getJob(jobId) {
+        const tableName = requireMainTableName();
+        const response = await dynamodb.send(new GetCommand({
+            TableName: tableName,
+            Key: jobKey(jobId)
+        }));
+        const job = response.Item ? itemToJob(response.Item) : null;
+        devLog("jobs.get", {
+            job_id: jobId,
+            found: Boolean(job),
+            customer_id: job?.customerId ?? null,
+            status: job?.status ?? null
+        });
+        return job;
+    }
+    async listChildJobs(parentJobId) {
+        const response = await dynamodb.send(new QueryCommand({
+            TableName: requireMainTableName(),
+            KeyConditionExpression: "pk = :pk and begins_with(sk, :prefix)",
+            ExpressionAttributeValues: {
+                ":pk": jobPk(parentJobId),
+                ":prefix": "CHILD#"
+            },
+            ScanIndexForward: false,
+            Limit: 500
+        }));
+        const childIds = (response.Items ?? [])
+            .map((item) => typeof item.child_job_id === "string" ? item.child_job_id : null)
+            .filter((id) => Boolean(id));
+        const jobs = [];
+        for (const childId of childIds) {
+            const child = await this.getJob(childId);
+            if (child) {
+                jobs.push(child);
+            }
+        }
+        return jobs;
+    }
+    async listJobs(input) {
+        const tableName = requireMainTableName();
+        const limit = Math.max(1, Math.min(input.limit ?? 100, 500));
+        const index = input.templateId ? "gsi2" : input.tracer ? "gsi3" : "gsi1";
+        const partition = input.templateId
+            ? customerTemplatePk(input.customerId, input.templateId)
+            : input.tracer
+                ? customerTracerPk(input.customerId, input.tracer)
+                : customerPk(input.customerId);
+        const response = await dynamodb.send(new QueryCommand({
+            TableName: tableName,
+            IndexName: index,
+            KeyConditionExpression: `${index}pk = :pk`,
+            ExpressionAttributeValues: {
+                ":pk": partition
+            },
+            ScanIndexForward: false,
+            Limit: limit,
+            ExclusiveStartKey: input.cursor ? cursorToStartKey(input, index) : undefined
+        }));
+        const jobs = (response.Items ?? []).map(itemToJob);
+        const filtered = jobs.filter((job) => {
+            if (input.tracer && job.tracer !== input.tracer) {
+                return false;
+            }
+            if (input.startTime && job.createdAt < input.startTime) {
+                return false;
+            }
+            if (input.endTime && job.createdAt > input.endTime) {
+                return false;
+            }
+            return true;
+        });
+        devLog("jobs.list", {
+            customer_id: input.customerId,
+            template_id: input.templateId ?? null,
+            tracer: input.tracer ?? null,
+            count: filtered.length,
+            limit
+        });
+        return filtered;
+    }
+    async cancelJob(jobId) {
+        const now = nowIso();
+        await this.addLogEvent({
+            id: createId("evt"),
+            jobId,
+            level: "warn",
+            message: "Job cancelled",
+            metadata: { reason: "user_request" },
+            progress: null,
+            artifactKey: null,
+            createdAt: now
+        });
+        await this.updateJobStatus({
+            id: jobId,
+            status: "cancelled",
+            error: { message: "Cancelled by user request." },
+            completedAt: now
+        });
+        devLog("jobs.cancelled", {
+            job_id: jobId
+        }, "warn");
+    }
+    async updateJobStatus(input) {
+        const current = await this.getJob(input.id);
+        const timestamp = nowIso();
+        const isTerminalStatus = input.status === "succeeded" || input.status === "failed" || input.status === "cancelled";
+        const shouldReleaseRootCapacity = Boolean(isTerminalStatus
+            && current
+            && current.parentJobId === null
+            && ACTIVE_ROOT_JOB_STATUSES.includes(current.status));
+        if (isTerminalStatus && current && !ACTIVE_ROOT_JOB_STATUSES.includes(current.status)) {
+            devLog("jobs.status.terminal_update_skipped", {
+                job_id: input.id,
+                current_status: current.status,
+                attempted_status: input.status
+            }, "warn");
+            return;
+        }
+        const nextCompletedAt = input.completedAt
+            ?? (isTerminalStatus ? timestamp : current?.completedAt ?? null);
+        const nextStartedAt = input.startedAt !== undefined
+            ? input.startedAt
+            : input.status === "running"
+                ? current?.startedAt ?? timestamp
+                : input.status === "queued"
+                    ? null
+                    : current?.startedAt ?? null;
+        const expressionAttributeValues = {
+            ":status": input.status,
+            ":progress": input.progress ?? current?.progress ?? 0,
+            ":result": input.result === undefined ? current?.result ?? null : input.result,
+            ":error": input.error === undefined ? current?.error ?? null : input.error,
+            ":runAfter": input.runAfter ?? current?.runAfter ?? timestamp,
+            ":startedAt": nextStartedAt,
+            ":completedAt": nextCompletedAt,
+            ":updatedAt": timestamp
+        };
+        const conditionExpression = isTerminalStatus
+            ? [
+                "#status = :queuedStatus",
+                "#status = :runningStatus",
+                "#status = :waitingForProviderStatus",
+                "#status = :waitingForChildStatus",
+                "#status = :waitingForHumanStatus"
+            ].join(" OR ")
+            : [
+                "attribute_not_exists(#status)",
+                "(#status <> :succeededStatus AND #status <> :failedStatus AND #status <> :cancelledStatus)"
+            ].join(" OR ");
+        if (isTerminalStatus) {
+            expressionAttributeValues[":queuedStatus"] = "queued";
+            expressionAttributeValues[":runningStatus"] = "running";
+            expressionAttributeValues[":waitingForProviderStatus"] = "waiting_for_provider";
+            expressionAttributeValues[":waitingForChildStatus"] = "waiting_for_child";
+            expressionAttributeValues[":waitingForHumanStatus"] = "waiting_for_human";
+        }
+        else {
+            expressionAttributeValues[":succeededStatus"] = "succeeded";
+            expressionAttributeValues[":failedStatus"] = "failed";
+            expressionAttributeValues[":cancelledStatus"] = "cancelled";
+        }
+        try {
+            await dynamodb.send(new UpdateCommand({
+                TableName: requireMainTableName(),
+                Key: jobKey(input.id),
+                UpdateExpression: [
+                    "set #status = :status",
+                    "progress = :progress",
+                    "result_json = :result",
+                    "error_json = :error",
+                    "run_after = :runAfter",
+                    "started_at = :startedAt",
+                    "completed_at = :completedAt",
+                    "updated_at = :updatedAt"
+                ].join(", "),
+                ConditionExpression: conditionExpression,
+                ExpressionAttributeNames: {
+                    "#status": "status"
+                },
+                ExpressionAttributeValues: expressionAttributeValues
+            }));
+        }
+        catch (error) {
+            if (isConditionalCheckFailed(error)) {
+                if (isTerminalStatus) {
+                    devLog("jobs.status.terminal_update_condition_failed", {
+                        job_id: input.id,
+                        attempted_status: input.status
+                    }, "warn");
+                    return;
+                }
+                devLog("jobs.status.non_terminal_update_skipped", {
+                    job_id: input.id,
+                    attempted_status: input.status,
+                    progress: input.progress ?? current?.progress ?? 0
+                }, "warn");
+                return;
+            }
+            throw error;
+        }
+        if (shouldReleaseRootCapacity && current) {
+            await this.decrementRootCapacity(current.customerId, current.id);
+        }
+        devLog("jobs.status.updated", {
+            job_id: input.id,
+            status: input.status,
+            progress: input.progress ?? current?.progress ?? 0,
+            completed_at: nextCompletedAt
+        }, input.status === "failed" ? "error" : input.status === "cancelled" ? "warn" : "info");
+    }
+    async insertArtifact(record) {
+        const createdAt = nowIso();
+        await dynamodb.send(new PutCommand({
+            TableName: requireMainTableName(),
+            Item: artifactToItem({
+                id: record.id,
+                jobId: record.jobId,
+                customerId: record.customerId,
+                templateId: record.templateId,
+                kind: record.kind,
+                storageKey: record.storageKey,
+                publicUrl: record.publicUrl ?? null,
+                metadata: record.metadata ?? {},
+                createdAt
+            })
+        }));
+        devLog("jobs.artifact.inserted", {
+            artifact_id: record.id,
+            job_id: record.jobId,
+            customer_id: record.customerId,
+            template_id: record.templateId,
+            kind: record.kind,
+            storage_key: record.storageKey,
+            has_public_url: Boolean(record.publicUrl)
+        });
+    }
+    async sweepStaleActiveJobs(input = {}) {
+        const customerIds = input.customerId
+            ? [input.customerId]
+            : await this.listCapacityCustomerIds(input.maxCustomers ?? 100);
+        let swept = 0;
+        let scanned = 0;
+        for (const customerId of customerIds) {
+            const result = await this.sweepStaleActiveJobsForCustomer(customerId, {
+                maxJobs: input.maxJobsPerCustomer ?? 1000
+            });
+            swept += result.swept;
+            scanned += result.scanned;
+        }
+        return {
+            customers: customerIds.length,
+            scanned,
+            swept
+        };
+    }
+    async listArtifactsForJob(input) {
+        const response = await dynamodb.send(new QueryCommand({
+            TableName: requireMainTableName(),
+            KeyConditionExpression: "pk = :pk and begins_with(sk, :prefix)",
+            ExpressionAttributeValues: {
+                ":pk": jobPk(input.jobId),
+                ":prefix": "ARTIFACT#"
+            },
+            ScanIndexForward: false,
+            Limit: Math.max(1, Math.min(input.limit ?? 100, 501))
+        }));
+        const artifacts = (response.Items ?? [])
+            .map(itemToArtifact)
+            .filter((artifact) => {
+            if (artifact.customerId !== input.customerId) {
+                return false;
+            }
+            if (input.templateId && artifact.templateId !== input.templateId) {
+                return false;
+            }
+            if (input.cursor && (artifact.createdAt > input.cursor.createdAt || (artifact.createdAt === input.cursor.createdAt && artifact.id >= input.cursor.id))) {
+                return false;
+            }
+            return true;
+        });
+        devLog("jobs.artifacts.list", {
+            job_id: input.jobId,
+            customer_id: input.customerId,
+            template_id: input.templateId ?? null,
+            count: artifacts.length,
+            limit: input.limit ?? 100
+        });
+        return artifacts;
+    }
+    async listLogs(input) {
+        const response = await dynamodb.send(new QueryCommand({
+            TableName: requireMainTableName(),
+            KeyConditionExpression: "pk = :pk and begins_with(sk, :prefix)",
+            ExpressionAttributeValues: {
+                ":pk": jobPk(input.jobId),
+                ":prefix": "LOG#"
+            },
+            ScanIndexForward: false,
+            Limit: Math.max(1, Math.min(input.limit ?? 100, 500))
+        }));
+        return (response.Items ?? [])
+            .map(itemToLog)
+            .filter((event) => {
+            if (input.startTime && event.createdAt < input.startTime) {
+                return false;
+            }
+            if (input.endTime && event.createdAt > input.endTime) {
+                return false;
+            }
+            if (input.cursor && (event.createdAt > input.cursor.createdAt || (event.createdAt === input.cursor.createdAt && event.id >= input.cursor.id))) {
+                return false;
+            }
+            return true;
+        });
+    }
+    async listLogStorageKeys() {
+        return [];
+    }
+    async getRootQueueCapacityStatus(customer) {
+        await this.sweepStaleActiveJobsForCustomer(customer.id);
+        await this.ensureRootCapacityCounter(customer.id);
+        const response = await dynamodb.send(new GetCommand({
+            TableName: requireMainTableName(),
+            Key: customerJobCapacityKey(customer.id)
+        }));
+        const activeCount = typeof response.Item?.active_count === "number" ? response.Item.active_count : 0;
+        const limit = config.MAX_ACTIVE_JOBS_PER_CUSTOMER;
+        return {
+            plan_tier: customer.planTier,
+            scope: "root_jobs",
+            customer: {
+                limit,
+                pending_count: activeCount,
+                active_count: activeCount,
+                remaining: Math.max(0, limit - activeCount),
+                enabled: true
+            },
+            pending_statuses: ACTIVE_ROOT_JOB_STATUSES,
+            retry_after_seconds: 30
+        };
+    }
+    async persistRootAndStart(tableName, job) {
+        await this.sweepStaleActiveJobsForCustomer(job.customerId);
+        await this.ensureRootCapacityCounter(job.customerId);
+        const queuedLogId = createId("evt");
+        for (let attempt = 0; attempt < 2; attempt += 1) {
+            try {
+                await this.writeRootJobTransaction(tableName, job, queuedLogId);
+                break;
+            }
+            catch (error) {
+                if (!isTransactionCancelledForCapacity(error)) {
+                    throw error;
+                }
+                const activeCount = await this.reconcileRootCapacityCounter(job.customerId);
+                if (attempt === 0 && activeCount < config.MAX_ACTIVE_JOBS_PER_CUSTOMER) {
+                    devLog("jobs.capacity.reconciled_retry", {
+                        job_id: job.id,
+                        customer_id: job.customerId,
+                        active_count: activeCount,
+                        limit: config.MAX_ACTIVE_JOBS_PER_CUSTOMER
+                    }, "warn");
+                    continue;
+                }
+                throw new ActiveJobLimitExceededError(`Customer has reached the active job limit of ${config.MAX_ACTIVE_JOBS_PER_CUSTOMER}.`, config.MAX_ACTIVE_JOBS_PER_CUSTOMER, activeCount, ACTIVE_ROOT_JOB_STATUSES, 30);
+            }
+        }
+        try {
+            const executionArn = await this.startWorkflow(job);
+            if (executionArn) {
+                await dynamodb.send(new UpdateCommand({
+                    TableName: tableName,
+                    Key: jobKey(job.id),
+                    UpdateExpression: "set step_execution_arn = :arn, updated_at = :updated",
+                    ExpressionAttributeValues: {
+                        ":arn": executionArn,
+                        ":updated": nowIso()
+                    }
+                }));
+            }
+        }
+        catch (error) {
+            await this.updateJobStatus({
+                id: job.id,
+                status: "failed",
+                error: {
+                    message: error instanceof Error ? error.message : "Unable to start job workflow.",
+                    type: "job_workflow_start_failed"
+                }
+            });
+            throw error;
+        }
+    }
+    async writeRootJobTransaction(tableName, job, queuedLogId) {
+        await dynamodb.send(new TransactWriteCommand({
+            TransactItems: [
+                {
+                    Update: {
+                        TableName: tableName,
+                        Key: customerJobCapacityKey(job.customerId),
+                        UpdateExpression: [
+                            "set active_count = if_not_exists(active_count, :zero) + :one",
+                            "updated_at = :updated",
+                            "capacity_limit = :limit",
+                            "entity_type = :entityType"
+                        ].join(", "),
+                        ConditionExpression: "attribute_not_exists(active_count) OR active_count < :limit",
+                        ExpressionAttributeValues: {
+                            ":zero": 0,
+                            ":one": 1,
+                            ":limit": config.MAX_ACTIVE_JOBS_PER_CUSTOMER,
+                            ":updated": job.createdAt,
+                            ":entityType": "CustomerJobCapacity"
+                        }
+                    }
+                },
+                {
+                    Put: {
+                        TableName: tableName,
+                        Item: jobToItem(job),
+                        ConditionExpression: "attribute_not_exists(pk)"
+                    }
+                },
+                {
+                    Put: {
+                        TableName: tableName,
+                        Item: {
+                            pk: jobPk(job.id),
+                            sk: `LOG#${job.createdAt}#${queuedLogId}`,
+                            entity_type: "JobLogEvent",
+                            id: queuedLogId,
+                            job_id: job.id,
+                            level: "info",
+                            message: "Job queued",
+                            metadata_json: {
+                                runtime: "serverless",
+                                template_id: job.templateId,
+                                operation_name: job.operationName
+                            },
+                            progress: 0,
+                            artifact_key: null,
+                            created_at: job.createdAt
+                        }
+                    }
+                }
+            ]
+        }));
+    }
+    async ensureRootCapacityCounter(customerId) {
+        const tableName = requireMainTableName();
+        const response = await dynamodb.send(new GetCommand({
+            TableName: tableName,
+            Key: customerJobCapacityKey(customerId)
+        }));
+        if (response.Item) {
+            return;
+        }
+        const activeCount = await this.countActiveRootJobs(customerId);
+        try {
+            await dynamodb.send(new PutCommand({
+                TableName: tableName,
+                Item: {
+                    ...customerJobCapacityKey(customerId),
+                    entity_type: "CustomerJobCapacity",
+                    customer_id: customerId,
+                    active_count: activeCount,
+                    capacity_limit: config.MAX_ACTIVE_JOBS_PER_CUSTOMER,
+                    updated_at: nowIso()
+                },
+                ConditionExpression: "attribute_not_exists(pk)"
+            }));
+        }
+        catch (error) {
+            if (isConditionalCheckFailed(error)) {
+                return;
+            }
+            throw error;
+        }
+    }
+    async reconcileRootCapacityCounter(customerId) {
+        await this.sweepStaleActiveJobsForCustomer(customerId);
+        const activeCount = await this.countActiveRootJobs(customerId);
+        await dynamodb.send(new UpdateCommand({
+            TableName: requireMainTableName(),
+            Key: customerJobCapacityKey(customerId),
+            UpdateExpression: "set active_count = :activeCount, capacity_limit = :limit, updated_at = :updated, entity_type = :entityType",
+            ExpressionAttributeValues: {
+                ":activeCount": activeCount,
+                ":limit": config.MAX_ACTIVE_JOBS_PER_CUSTOMER,
+                ":updated": nowIso(),
+                ":entityType": "CustomerJobCapacity"
+            }
+        }));
+        return activeCount;
+    }
+    async getRootCapacityCount(customerId) {
+        const response = await dynamodb.send(new GetCommand({
+            TableName: requireMainTableName(),
+            Key: customerJobCapacityKey(customerId)
+        }));
+        return typeof response.Item?.active_count === "number" ? response.Item.active_count : null;
+    }
+    async countActiveRootJobs(customerId) {
+        let count = 0;
+        let exclusiveStartKey;
+        do {
+            const response = await dynamodb.send(new QueryCommand({
+                TableName: requireMainTableName(),
+                IndexName: "gsi1",
+                KeyConditionExpression: "gsi1pk = :pk",
+                ExpressionAttributeValues: {
+                    ":pk": customerPk(customerId)
+                },
+                ScanIndexForward: false,
+                Limit: 500,
+                ExclusiveStartKey: exclusiveStartKey
+            }));
+            for (const item of response.Items ?? []) {
+                const job = itemToJob(item);
+                if (job.parentJobId === null && ACTIVE_ROOT_JOB_STATUSES.includes(job.status)) {
+                    count += 1;
+                }
+            }
+            exclusiveStartKey = response.LastEvaluatedKey;
+        } while (exclusiveStartKey);
+        return count;
+    }
+    async listCapacityCustomerIds(limit) {
+        const customerIds = [];
+        let exclusiveStartKey;
+        do {
+            const response = await dynamodb.send(new ScanCommand({
+                TableName: requireMainTableName(),
+                FilterExpression: "entity_type = :entityType",
+                ExpressionAttributeValues: {
+                    ":entityType": "CustomerJobCapacity"
+                },
+                Limit: Math.max(1, Math.min(limit - customerIds.length, 100)),
+                ExclusiveStartKey: exclusiveStartKey
+            }));
+            for (const item of response.Items ?? []) {
+                const customerId = typeof item.customer_id === "string"
+                    ? item.customer_id
+                    : parseCustomerIdFromCapacityPk(item.pk);
+                if (customerId) {
+                    customerIds.push(customerId);
+                }
+                if (customerIds.length >= limit) {
+                    break;
+                }
+            }
+            exclusiveStartKey = customerIds.length >= limit
+                ? undefined
+                : response.LastEvaluatedKey;
+        } while (exclusiveStartKey);
+        return customerIds;
+    }
+    async sweepStaleActiveJobsForCustomer(customerId, input = {}) {
+        const maxJobs = Math.max(1, input.maxJobs ?? 1000);
+        let scanned = 0;
+        let swept = 0;
+        let exclusiveStartKey;
+        do {
+            const response = await dynamodb.send(new QueryCommand({
+                TableName: requireMainTableName(),
+                IndexName: "gsi1",
+                KeyConditionExpression: "gsi1pk = :pk",
+                ExpressionAttributeValues: {
+                    ":pk": customerPk(customerId)
+                },
+                ScanIndexForward: false,
+                Limit: Math.min(500, maxJobs - scanned),
+                ExclusiveStartKey: exclusiveStartKey
+            }));
+            for (const item of response.Items ?? []) {
+                scanned += 1;
+                const job = itemToJob(item);
+                const stale = describeStaleActiveJob(job);
+                if (!stale) {
+                    continue;
+                }
+                await this.failStaleActiveJob(job, stale);
+                swept += 1;
+            }
+            exclusiveStartKey = scanned >= maxJobs
+                ? undefined
+                : response.LastEvaluatedKey;
+        } while (exclusiveStartKey);
+        if (swept > 0) {
+            await this.reconcileRootCapacityCounterAfterSweep(customerId);
+        }
+        return { scanned, swept };
+    }
+    async failStaleActiveJob(job, stale) {
+        const now = nowIso();
+        await this.addLogEvent({
+            id: createId("evt"),
+            jobId: job.id,
+            level: "error",
+            message: "Job marked failed by stale-job sweeper",
+            metadata: {
+                type: "stale_job_swept",
+                stale_reason: stale.reason,
+                stale_threshold_seconds: stale.thresholdSeconds,
+                stale_age_seconds: stale.ageSeconds,
+                previous_status: job.status,
+                started_at: job.startedAt,
+                updated_at: job.updatedAt,
+                created_at: job.createdAt
+            },
+            progress: job.progress,
+            artifactKey: null,
+            createdAt: now
+        });
+        await this.updateJobStatus({
+            id: job.id,
+            status: "failed",
+            error: {
+                message: `Job was marked failed because it became stale: ${stale.reason}.`,
+                type: "stale_job_swept",
+                stale_reason: stale.reason,
+                stale_threshold_seconds: stale.thresholdSeconds,
+                stale_age_seconds: stale.ageSeconds
+            },
+            completedAt: now
+        });
+    }
+    async reconcileRootCapacityCounterAfterSweep(customerId) {
+        const activeCount = await this.countActiveRootJobs(customerId);
+        await dynamodb.send(new UpdateCommand({
+            TableName: requireMainTableName(),
+            Key: customerJobCapacityKey(customerId),
+            UpdateExpression: "set active_count = :activeCount, capacity_limit = :limit, updated_at = :updated, entity_type = :entityType",
+            ExpressionAttributeValues: {
+                ":activeCount": activeCount,
+                ":limit": config.MAX_ACTIVE_JOBS_PER_CUSTOMER,
+                ":updated": nowIso(),
+                ":entityType": "CustomerJobCapacity"
+            }
+        }));
+    }
+    async decrementRootCapacity(customerId, jobId) {
+        try {
+            await dynamodb.send(new UpdateCommand({
+                TableName: requireMainTableName(),
+                Key: customerJobCapacityKey(customerId),
+                UpdateExpression: "set active_count = active_count - :one, updated_at = :updated",
+                ConditionExpression: "active_count >= :one",
+                ExpressionAttributeValues: {
+                    ":one": 1,
+                    ":updated": nowIso()
+                }
+            }));
+            devLog("jobs.capacity.decremented", {
+                job_id: jobId,
+                customer_id: customerId
+            });
+        }
+        catch (error) {
+            if (isConditionalCheckFailed(error)) {
+                devLog("jobs.capacity.decrement_skipped", {
+                    job_id: jobId,
+                    customer_id: customerId
+                }, "warn");
+                return;
+            }
+            throw error;
+        }
+    }
+    async startWorkflow(job) {
+        if (!config.VIDFARM_JOB_STATE_MACHINE_ARN) {
+            return null;
+        }
+        const response = await sfn.send(new StartExecutionCommand({
+            stateMachineArn: config.VIDFARM_JOB_STATE_MACHINE_ARN,
+            name: job.id.replace(/[^A-Za-z0-9-_]/g, "-").slice(0, 80),
+            input: JSON.stringify({
+                job_id: job.id,
+                customer_id: job.customerId,
+                template_id: job.templateId,
+                operation_name: job.operationName,
+                workflow_name: job.workflowName,
+                stage: "run",
+                attempt: 1
+            })
+        }));
+        devLog("jobs.workflow.started", {
+            job_id: job.id,
+            execution_arn: response.executionArn ?? null,
+            state_machine_arn: config.VIDFARM_JOB_STATE_MACHINE_ARN
+        });
+        return response.executionArn ?? null;
+    }
+    async addLogEvent(event) {
+        await dynamodb.send(new PutCommand({
+            TableName: requireMainTableName(),
+            Item: {
+                pk: jobPk(event.jobId),
+                sk: `LOG#${event.createdAt}#${event.id}`,
+                entity_type: "JobLogEvent",
+                id: event.id,
+                job_id: event.jobId,
+                level: event.level,
+                message: event.message,
+                metadata_json: event.metadata,
+                progress: event.progress,
+                artifact_key: event.artifactKey,
+                created_at: event.createdAt
+            }
+        }));
+    }
+}
+function isConditionalCheckFailed(error) {
+    return Boolean(error
+        && typeof error === "object"
+        && "name" in error
+        && error.name === "ConditionalCheckFailedException");
+}
+function isTransactionCancelledForCapacity(error) {
+    if (!error || typeof error !== "object" || !("name" in error) || error.name !== "TransactionCanceledException") {
+        return false;
+    }
+    const reasons = "CancellationReasons" in error ? error.CancellationReasons : null;
+    return Array.isArray(reasons) && reasons.some((reason) => {
+        return reason && typeof reason === "object" && "Code" in reason && reason.Code === "ConditionalCheckFailed";
+    });
+}
+function requireMainTableName() {
+    if (config.RECORDS_DRIVER === "local") {
+        return config.VIDFARM_SERVERLESS_MAIN_TABLE_NAME || "vidfarm-local-main";
+    }
+    if (!config.VIDFARM_SERVERLESS_MAIN_TABLE_NAME) {
+        throw new Error("Serverless job store requires VIDFARM_SERVERLESS_MAIN_TABLE_NAME.");
+    }
+    return config.VIDFARM_SERVERLESS_MAIN_TABLE_NAME;
+}
+function normalizeWebhookUrl(value) {
+    if (typeof value !== "string") {
+        return null;
+    }
+    const trimmed = value.trim();
+    return trimmed ? trimmed : null;
+}
+function jobPk(jobId) {
+    return `JOB#${jobId}`;
+}
+function jobKey(jobId) {
+    return {
+        pk: jobPk(jobId),
+        sk: "META"
+    };
+}
+function customerPk(customerId) {
+    return `CUSTOMER#${customerId}`;
+}
+function customerJobCapacityKey(customerId) {
+    return {
+        pk: customerPk(customerId),
+        sk: "JOB_CAPACITY"
+    };
+}
+function parseCustomerIdFromCapacityPk(value) {
+    if (typeof value !== "string" || !value.startsWith("CUSTOMER#")) {
+        return null;
+    }
+    return value.slice("CUSTOMER#".length) || null;
+}
+function describeStaleActiveJob(job) {
+    if (!ACTIVE_JOB_STATUSES.has(job.status)) {
+        return null;
+    }
+    const wallClockAgeSeconds = secondsSince(job.createdAt);
+    if (wallClockAgeSeconds >= config.MAX_JOB_WALL_CLOCK_SECONDS) {
+        return {
+            reason: "wall_clock_timeout",
+            thresholdSeconds: config.MAX_JOB_WALL_CLOCK_SECONDS,
+            ageSeconds: wallClockAgeSeconds
+        };
+    }
+    if (job.status === "running") {
+        const heartbeatAt = job.updatedAt || job.startedAt || job.createdAt;
+        const heartbeatAgeSeconds = secondsSince(heartbeatAt);
+        if (heartbeatAgeSeconds >= config.STALE_RUNNING_JOB_SECONDS) {
+            return {
+                reason: "running_heartbeat_timeout",
+                thresholdSeconds: config.STALE_RUNNING_JOB_SECONDS,
+                ageSeconds: heartbeatAgeSeconds
+            };
+        }
+    }
+    if (job.status === "queued") {
+        const runAfterAgeSeconds = secondsSince(job.runAfter);
+        if (runAfterAgeSeconds >= config.MAX_JOB_WALL_CLOCK_SECONDS) {
+            return {
+                reason: "queued_run_after_timeout",
+                thresholdSeconds: config.MAX_JOB_WALL_CLOCK_SECONDS,
+                ageSeconds: runAfterAgeSeconds
+            };
+        }
+    }
+    return null;
+}
+function secondsSince(value) {
+    const timestamp = value ? Date.parse(value) : Number.NaN;
+    const effectiveTimestamp = Number.isFinite(timestamp) ? timestamp : Date.now();
+    return Math.max(0, Math.floor((Date.now() - effectiveTimestamp) / 1000));
+}
+function customerTemplatePk(customerId, templateId) {
+    return `CUSTOMER#${customerId}#TEMPLATE#${templateId}`;
+}
+function customerTracerPk(customerId, tracer) {
+    return `CUSTOMER#${customerId}#TRACER#${tracer}`;
+}
+function jobSortKey(job) {
+    return `JOB#${job.createdAt}#${job.id}`;
+}
+function jobToItem(job) {
+    return {
+        pk: jobPk(job.id),
+        sk: "META",
+        entity_type: "Job",
+        gsi1pk: customerPk(job.customerId),
+        gsi1sk: jobSortKey(job),
+        gsi2pk: customerTemplatePk(job.customerId, job.templateId),
+        gsi2sk: jobSortKey(job),
+        gsi3pk: customerTracerPk(job.customerId, job.tracer),
+        gsi3sk: jobSortKey(job),
+        job_id: job.id,
+        template_id: job.templateId,
+        operation_name: job.operationName,
+        workflow_name: job.workflowName,
+        tracer: job.tracer,
+        status: job.status,
+        customer_id: job.customerId,
+        payload_json: job.payload,
+        result_json: job.result,
+        error_json: job.error,
+        progress: job.progress,
+        webhook_url: job.webhookUrl,
+        parent_job_id: job.parentJobId,
+        priority: job.priority,
+        attempt_count: job.attemptCount,
+        max_attempts: job.maxAttempts,
+        run_after: job.runAfter,
+        created_at: job.createdAt,
+        updated_at: job.updatedAt,
+        started_at: job.startedAt,
+        completed_at: job.completedAt
+    };
+}
+function itemToJob(item) {
+    return {
+        id: String(item.job_id),
+        templateId: String(item.template_id),
+        operationName: String(item.operation_name),
+        workflowName: String(item.workflow_name),
+        tracer: String(item.tracer),
+        status: item.status,
+        customerId: String(item.customer_id),
+        payload: asRecord(item.payload_json),
+        result: item.result_json == null ? null : asRecord(item.result_json),
+        error: item.error_json == null ? null : asRecord(item.error_json),
+        progress: typeof item.progress === "number" ? item.progress : 0,
+        webhookUrl: typeof item.webhook_url === "string" ? item.webhook_url : null,
+        parentJobId: typeof item.parent_job_id === "string" ? item.parent_job_id : null,
+        priority: typeof item.priority === "number" ? item.priority : 100,
+        attemptCount: typeof item.attempt_count === "number" ? item.attempt_count : 0,
+        maxAttempts: typeof item.max_attempts === "number" ? item.max_attempts : 6,
+        runAfter: String(item.run_after),
+        createdAt: String(item.created_at),
+        updatedAt: String(item.updated_at),
+        startedAt: typeof item.started_at === "string" ? item.started_at : null,
+        completedAt: typeof item.completed_at === "string" ? item.completed_at : null
+    };
+}
+function itemToLog(item) {
+    return {
+        id: String(item.id),
+        jobId: String(item.job_id),
+        level: item.level,
+        message: String(item.message),
+        metadata: asRecord(item.metadata_json),
+        progress: typeof item.progress === "number" ? item.progress : null,
+        artifactKey: typeof item.artifact_key === "string" ? item.artifact_key : null,
+        createdAt: String(item.created_at)
+    };
+}
+function artifactToItem(artifact) {
+    return {
+        pk: jobPk(artifact.jobId),
+        sk: `ARTIFACT#${artifact.createdAt}#${artifact.id}`,
+        entity_type: "Artifact",
+        id: artifact.id,
+        artifact_id: artifact.id,
+        job_id: artifact.jobId,
+        customer_id: artifact.customerId,
+        template_id: artifact.templateId,
+        kind: artifact.kind,
+        storage_key: artifact.storageKey,
+        public_url: artifact.publicUrl,
+        metadata_json: artifact.metadata,
+        created_at: artifact.createdAt
+    };
+}
+function itemToArtifact(item) {
+    return {
+        id: String(item.artifact_id ?? item.id),
+        jobId: String(item.job_id),
+        customerId: String(item.customer_id),
+        templateId: String(item.template_id),
+        kind: String(item.kind),
+        storageKey: String(item.storage_key),
+        publicUrl: typeof item.public_url === "string" ? item.public_url : null,
+        metadata: asRecord(item.metadata_json),
+        createdAt: String(item.created_at)
+    };
+}
+function asRecord(value) {
+    return value && typeof value === "object" && !Array.isArray(value)
+        ? value
+        : {};
+}
+function cursorToStartKey(input, index) {
+    const cursor = input.cursor;
+    if (!cursor) {
+        return undefined;
+    }
+    const syntheticJob = { id: cursor.id, createdAt: cursor.createdAt };
+    return {
+        pk: jobPk(cursor.id),
+        sk: "META",
+        [`${index}pk`]: input.templateId
+            ? customerTemplatePk(input.customerId, input.templateId)
+            : input.tracer
+                ? customerTracerPk(input.customerId, input.tracer)
+                : customerPk(input.customerId),
+        [`${index}sk`]: jobSortKey(syntheticJob)
+    };
+}
+//# sourceMappingURL=serverless-jobs.js.map