@mevdragon/vidfarm-devcli 0.5.2 → 0.6.0

package/dist/src/services/serverless-provider-keys.js

@@ -0,0 +1,409 @@
+import { DynamoDBClient } from "@aws-sdk/client-dynamodb";
+import { DeleteCommand, DynamoDBDocumentClient, PutCommand, QueryCommand, UpdateCommand } from "@aws-sdk/lib-dynamodb";
+import { config } from "../config.js";
+import { devLog } from "../lib/dev-log.js";
+import { nowIso } from "../lib/time.js";
+import { createLocalDynamoClient } from "./local-dynamo.js";
+// `vidfarm serve` (RECORDS_DRIVER=local) swaps in the disk-backed shim so
+// provider keys live on disk; deployed environments keep the real client.
+const dynamodb = config.RECORDS_DRIVER === "local"
+    ? createLocalDynamoClient()
+    : DynamoDBDocumentClient.from(new DynamoDBClient({}), {
+        marshallOptions: {
+            removeUndefinedValues: true
+        }
+    });
+export class ServerlessProviderKeyService {
+    async listProviderKeys(customerId) {
+        const items = await this.queryCustomerKeys(customerId);
+        devLog("provider_keys.list", {
+            customer_id: customerId,
+            count: items.length
+        });
+        return items.map(itemToPublicProviderKey);
+    }
+    async listProviderKeysWithSecrets(customerId) {
+        const items = await this.queryCustomerKeys(customerId);
+        return items.map(itemToSecretProviderKey);
+    }
+    async createProviderKey(record) {
+        const timestamp = nowIso();
+        await dynamodb.send(new PutCommand({
+            TableName: requireMainTableName(),
+            Item: {
+                pk: providerKeyPk(record.id),
+                sk: "META",
+                entity_type: "ProviderKey",
+                gsi1pk: customerProviderKeysPk(record.customerId),
+                gsi1sk: providerKeySortKey(record.provider, timestamp, record.id),
+                key_id: record.id,
+                customer_id: record.customerId,
+                provider: record.provider,
+                label: record.label,
+                encrypted_secret: record.encryptedSecret,
+                status: "active",
+                last_used_at: null,
+                cooldown_until: null,
+                disabled_reason: null,
+                created_at: timestamp,
+                updated_at: timestamp
+            },
+            ConditionExpression: "attribute_not_exists(pk)"
+        }));
+        devLog("provider_keys.created", {
+            key_id: record.id,
+            customer_id: record.customerId,
+            provider: record.provider,
+            label: record.label
+        });
+    }
+    async deleteProviderKey(customerId, keyId) {
+        await dynamodb.send(new DeleteCommand({
+            TableName: requireMainTableName(),
+            Key: providerKeyKey(keyId),
+            ConditionExpression: "customer_id = :customerId",
+            ExpressionAttributeValues: {
+                ":customerId": customerId
+            }
+        })).catch((error) => {
+            if (isConditionalCheckFailed(error)) {
+                return;
+            }
+            throw error;
+        });
+        devLog("provider_keys.deleted", {
+            key_id: keyId,
+            customer_id: customerId
+        }, "warn");
+    }
+    async listAvailableProviderKeyCounts(customerId) {
+        const now = nowIso();
+        const counts = new Map();
+        for (const key of await this.listProviderKeysWithSecrets(customerId)) {
+            if (key.status !== "active" || !key.secret.trim()) {
+                continue;
+            }
+            if (key.cooldown_until && key.cooldown_until > now) {
+                continue;
+            }
+            const item = await this.getProviderKeyItem(customerId, key.id);
+            if (!item) {
+                continue;
+            }
+            const leaseExpiresAt = typeof item.active_lease_expires_at === "string" ? item.active_lease_expires_at : null;
+            if (leaseExpiresAt && leaseExpiresAt > now) {
+                continue;
+            }
+            counts.set(key.provider, (counts.get(key.provider) ?? 0) + 1);
+        }
+        return [...counts.entries()].map(([provider, availableCount]) => ({
+            customerId,
+            provider,
+            availableCount
+        })).map((entry) => {
+            devLog("provider_keys.available_count", {
+                customer_id: entry.customerId,
+                provider: entry.provider,
+                available_count: entry.availableCount
+            });
+            return entry;
+        });
+    }
+    async acquireProviderKeyLease(input) {
+        const now = nowIso();
+        const keys = (await this.queryCustomerKeys(input.customerId))
+            .map(itemToSecretProviderKeyWithLease)
+            .filter((key) => key.provider === input.provider)
+            .filter((key) => key.status === "active")
+            .filter((key) => !key.cooldown_until || key.cooldown_until <= now)
+            .filter((key) => key.secret.trim().length > 0)
+            .sort((a, b) => {
+            const aLast = a.last_used_at ?? "";
+            const bLast = b.last_used_at ?? "";
+            if (!aLast && bLast)
+                return -1;
+            if (aLast && !bLast)
+                return 1;
+            return aLast.localeCompare(bLast) || a.created_at.localeCompare(b.created_at);
+        });
+        for (const key of keys) {
+            try {
+                await dynamodb.send(new UpdateCommand({
+                    TableName: requireMainTableName(),
+                    Key: providerKeyKey(key.id),
+                    ConditionExpression: [
+                        "customer_id = :customerId",
+                        "#status = :active",
+                        "(attribute_not_exists(cooldown_until) OR attribute_type(cooldown_until, :nullType) OR cooldown_until = :empty OR cooldown_until <= :now)",
+                        "(attribute_not_exists(active_lease_expires_at) OR active_lease_expires_at <= :now OR active_lease_token = :leaseToken)"
+                    ].join(" AND "),
+                    UpdateExpression: [
+                        "set active_lease_token = :leaseToken",
+                        "active_lease_worker_id = :workerId",
+                        "active_lease_job_id = :jobId",
+                        "active_lease_expires_at = :expiresAt",
+                        "updated_at = :now"
+                    ].join(", "),
+                    ExpressionAttributeNames: {
+                        "#status": "status"
+                    },
+                    ExpressionAttributeValues: {
+                        ":customerId": input.customerId,
+                        ":active": "active",
+                        ":nullType": "NULL",
+                        ":empty": "",
+                        ":now": now,
+                        ":leaseToken": input.leaseToken,
+                        ":workerId": input.workerId,
+                        ":jobId": input.jobId,
+                        ":expiresAt": input.expiresAt
+                    }
+                }));
+                await this.putLeaseEvent({
+                    keyId: key.id,
+                    leaseToken: input.leaseToken,
+                    workerId: input.workerId,
+                    jobId: input.jobId,
+                    expiresAt: input.expiresAt
+                });
+                devLog("provider_keys.lease.acquired", {
+                    key_id: key.id,
+                    customer_id: input.customerId,
+                    provider: input.provider,
+                    job_id: input.jobId,
+                    worker_id: input.workerId,
+                    expires_at: input.expiresAt
+                });
+                return {
+                    keyId: key.id,
+                    encryptedSecret: key.secret
+                };
+            }
+            catch (error) {
+                if (isConditionalCheckFailed(error)) {
+                    continue;
+                }
+                throw error;
+            }
+        }
+        devLog("provider_keys.lease.unavailable", {
+            customer_id: input.customerId,
+            provider: input.provider,
+            job_id: input.jobId,
+            worker_id: input.workerId
+        }, "warn");
+        return null;
+    }
+    async releaseProviderKeyLease(input) {
+        const now = nowIso();
+        await dynamodb.send(new UpdateCommand({
+            TableName: requireMainTableName(),
+            Key: providerKeyKey(input.keyId),
+            ConditionExpression: "active_lease_token = :leaseToken",
+            UpdateExpression: "SET updated_at = :now REMOVE active_lease_token, active_lease_worker_id, active_lease_job_id, active_lease_expires_at",
+            ExpressionAttributeValues: {
+                ":leaseToken": input.leaseToken,
+                ":now": now
+            }
+        })).catch((error) => {
+            if (isConditionalCheckFailed(error)) {
+                return;
+            }
+            throw error;
+        });
+        await dynamodb.send(new DeleteCommand({
+            TableName: requireMainTableName(),
+            Key: {
+                pk: providerKeyPk(input.keyId),
+                sk: `LEASE#${input.leaseToken}`
+            }
+        }));
+        devLog("provider_keys.lease.released", {
+            key_id: input.keyId
+        });
+    }
+    async extendProviderKeyLease(input) {
+        await dynamodb.send(new UpdateCommand({
+            TableName: requireMainTableName(),
+            Key: providerKeyKey(input.keyId),
+            ConditionExpression: "active_lease_token = :leaseToken",
+            UpdateExpression: "set active_lease_expires_at = :expiresAt, updated_at = :now",
+            ExpressionAttributeValues: {
+                ":leaseToken": input.leaseToken,
+                ":expiresAt": input.expiresAt,
+                ":now": nowIso()
+            }
+        })).catch((error) => {
+            if (isConditionalCheckFailed(error)) {
+                return;
+            }
+            throw error;
+        });
+        devLog("provider_keys.lease.extended", {
+            key_id: input.keyId,
+            expires_at: input.expiresAt
+        });
+    }
+    async recordProviderKeyUsage(input) {
+        const timestamp = nowIso();
+        await dynamodb.send(new PutCommand({
+            TableName: requireMainTableName(),
+            Item: {
+                pk: providerKeyPk(input.keyId),
+                sk: `USAGE#${timestamp}#${input.id}`,
+                entity_type: "ProviderKeyUsage",
+                gsi1pk: `JOB#${input.jobId}#PROVIDER_USAGE`,
+                gsi1sk: `USAGE#${timestamp}#${input.id}`,
+                usage_id: input.id,
+                key_id: input.keyId,
+                job_id: input.jobId,
+                provider: input.provider,
+                model: input.model ?? null,
+                event_type: input.eventType,
+                input_tokens: input.inputTokens ?? null,
+                output_tokens: input.outputTokens ?? null,
+                cost_usd: input.costUsd ?? null,
+                metadata: input.metadata ?? {},
+                created_at: timestamp
+            }
+        }));
+        devLog("provider_keys.usage.recorded", {
+            usage_id: input.id,
+            key_id: input.keyId,
+            job_id: input.jobId,
+            provider: input.provider,
+            model: input.model ?? null,
+            event_type: input.eventType,
+            cost_usd: input.costUsd ?? null
+        });
+    }
+    async touchProviderKey(keyId) {
+        const timestamp = nowIso();
+        await dynamodb.send(new UpdateCommand({
+            TableName: requireMainTableName(),
+            Key: providerKeyKey(keyId),
+            UpdateExpression: "set last_used_at = :now, updated_at = :now",
+            ExpressionAttributeValues: {
+                ":now": timestamp
+            }
+        }));
+    }
+    async setProviderKeyCooldown(keyId, cooldownUntil, status = "active", disabledReason = null) {
+        const timestamp = nowIso();
+        await dynamodb.send(new UpdateCommand({
+            TableName: requireMainTableName(),
+            Key: providerKeyKey(keyId),
+            UpdateExpression: "set cooldown_until = :cooldownUntil, #status = :status, disabled_reason = :disabledReason, updated_at = :now",
+            ExpressionAttributeNames: {
+                "#status": "status"
+            },
+            ExpressionAttributeValues: {
+                ":cooldownUntil": cooldownUntil instanceof Date ? cooldownUntil.toISOString() : cooldownUntil,
+                ":status": status,
+                ":disabledReason": disabledReason,
+                ":now": timestamp
+            }
+        }));
+    }
+    async queryCustomerKeys(customerId) {
+        const response = await dynamodb.send(new QueryCommand({
+            TableName: requireMainTableName(),
+            IndexName: "gsi1",
+            KeyConditionExpression: "gsi1pk = :pk",
+            ExpressionAttributeValues: {
+                ":pk": customerProviderKeysPk(customerId)
+            }
+        }));
+        return response.Items ?? [];
+    }
+    async getProviderKeyItem(customerId, keyId) {
+        const response = await dynamodb.send(new QueryCommand({
+            TableName: requireMainTableName(),
+            KeyConditionExpression: "pk = :pk and sk = :sk",
+            FilterExpression: "customer_id = :customerId",
+            ExpressionAttributeValues: {
+                ":pk": providerKeyPk(keyId),
+                ":sk": "META",
+                ":customerId": customerId
+            },
+            Limit: 1
+        }));
+        return response.Items?.[0] ?? null;
+    }
+    async putLeaseEvent(input) {
+        await dynamodb.send(new PutCommand({
+            TableName: requireMainTableName(),
+            Item: {
+                pk: providerKeyPk(input.keyId),
+                sk: `LEASE#${input.leaseToken}`,
+                entity_type: "ProviderKeyLease",
+                key_id: input.keyId,
+                lease_token: input.leaseToken,
+                worker_id: input.workerId,
+                job_id: input.jobId,
+                leased_at: nowIso(),
+                expires_at: input.expiresAt,
+                expires_at_epoch_seconds: Math.ceil(Date.parse(input.expiresAt) / 1000)
+            }
+        }));
+    }
+}
+function requireMainTableName() {
+    if (config.RECORDS_DRIVER === "local") {
+        return config.VIDFARM_SERVERLESS_MAIN_TABLE_NAME || "vidfarm-local-main";
+    }
+    if (!config.VIDFARM_SERVERLESS_MAIN_TABLE_NAME) {
+        throw new Error("Serverless provider key store requires VIDFARM_SERVERLESS_MAIN_TABLE_NAME.");
+    }
+    return config.VIDFARM_SERVERLESS_MAIN_TABLE_NAME;
+}
+function providerKeyPk(keyId) {
+    return `PROVIDER_KEY#${keyId}`;
+}
+function providerKeyKey(keyId) {
+    return {
+        pk: providerKeyPk(keyId),
+        sk: "META"
+    };
+}
+function customerProviderKeysPk(customerId) {
+    return `CUSTOMER#${customerId}#PROVIDER_KEYS`;
+}
+function providerKeySortKey(provider, createdAt, keyId) {
+    return `PROVIDER#${provider}#${createdAt}#${keyId}`;
+}
+function itemToPublicProviderKey(item) {
+    return {
+        id: String(item.key_id),
+        provider: normalizeProvider(item.provider),
+        label: typeof item.label === "string" ? item.label : null,
+        status: typeof item.status === "string" ? item.status : "active",
+        last_used_at: typeof item.last_used_at === "string" ? item.last_used_at : null,
+        cooldown_until: typeof item.cooldown_until === "string" ? item.cooldown_until : null,
+        disabled_reason: typeof item.disabled_reason === "string" ? item.disabled_reason : null,
+        created_at: String(item.created_at),
+        updated_at: String(item.updated_at)
+    };
+}
+function itemToSecretProviderKey(item) {
+    return {
+        ...itemToPublicProviderKey(item),
+        secret: typeof item.encrypted_secret === "string" ? item.encrypted_secret : ""
+    };
+}
+function itemToSecretProviderKeyWithLease(item) {
+    return {
+        ...itemToSecretProviderKey(item),
+        active_lease_expires_at: typeof item.active_lease_expires_at === "string" ? item.active_lease_expires_at : null
+    };
+}
+function normalizeProvider(value) {
+    if (value === "openai" || value === "gemini" || value === "openrouter" || value === "perplexity") {
+        return value;
+    }
+    return "openai";
+}
+function isConditionalCheckFailed(error) {
+    return Boolean(error && typeof error === "object" && "name" in error && error.name === "ConditionalCheckFailedException");
+}
+//# sourceMappingURL=serverless-provider-keys.js.map