@mevdragon/vidfarm-devcli 0.1.0

package/dist/src/db.js

@@ -0,0 +1,588 @@
+import Database from "better-sqlite3";
+import { config } from "./config.js";
+import { parseJson, stringifyJson } from "./lib/json.js";
+import { nowIso } from "./lib/time.js";
+const db = new Database(config.VIDFARM_DB_PATH);
+db.pragma("journal_mode = WAL");
+db.pragma("foreign_keys = ON");
+db.exec(`
+create table if not exists customers (
+  id text primary key,
+  email text not null unique,
+  name text,
+  default_webhook_url text,
+  created_at text not null,
+  updated_at text not null
+);
+
+create table if not exists api_keys (
+  id text primary key,
+  customer_id text not null references customers(id) on delete cascade,
+  key_hash text not null unique,
+  label text,
+  status text not null,
+  last_used_at text,
+  created_at text not null,
+  updated_at text not null
+);
+
+create table if not exists otp_challenges (
+  id text primary key,
+  email text not null,
+  code_hash text not null,
+  expires_at text not null,
+  consumed_at text,
+  created_at text not null
+);
+
+create table if not exists customer_provider_keys (
+  id text primary key,
+  customer_id text not null references customers(id) on delete cascade,
+  provider text not null,
+  label text,
+  encrypted_secret text not null,
+  status text not null,
+  weight integer not null default 1,
+  last_used_at text,
+  cooldown_until text,
+  disabled_reason text,
+  created_at text not null,
+  updated_at text not null
+);
+
+create table if not exists provider_key_leases (
+  key_id text primary key references customer_provider_keys(id) on delete cascade,
+  lease_token text not null,
+  worker_id text not null,
+  job_id text not null,
+  leased_at text not null,
+  expires_at text not null
+);
+
+create table if not exists provider_key_usage_events (
+  id text primary key,
+  key_id text not null references customer_provider_keys(id) on delete cascade,
+  job_id text not null,
+  provider text not null,
+  model text,
+  event_type text not null,
+  input_tokens integer,
+  output_tokens integer,
+  cost_usd real,
+  metadata text,
+  created_at text not null
+);
+
+create table if not exists template_configs (
+  id text primary key,
+  customer_id text not null references customers(id) on delete cascade,
+  template_id text not null,
+  config_json text not null,
+  created_at text not null,
+  updated_at text not null,
+  unique(customer_id, template_id)
+);
+
+create table if not exists jobs (
+  id text primary key,
+  template_id text not null,
+  operation_name text not null,
+  workflow_name text not null,
+  tracer text not null,
+  status text not null,
+  customer_id text not null references customers(id) on delete cascade,
+  payload_json text not null,
+  result_json text,
+  error_json text,
+  progress real not null default 0,
+  webhook_url text,
+  parent_job_id text references jobs(id) on delete set null,
+  priority integer not null default 100,
+  attempt_count integer not null default 0,
+  max_attempts integer not null default 6,
+  run_after text not null,
+  created_at text not null,
+  updated_at text not null,
+  started_at text,
+  completed_at text
+);
+
+create index if not exists idx_jobs_runnable on jobs(status, run_after, priority, created_at);
+
+create table if not exists job_events (
+  id text primary key,
+  job_id text not null references jobs(id) on delete cascade,
+  level text not null,
+  message text not null,
+  metadata_json text not null,
+  progress real,
+  artifact_key text,
+  created_at text not null
+);
+
+create index if not exists idx_job_events_job_time on job_events(job_id, created_at);
+
+create table if not exists artifacts (
+  id text primary key,
+  job_id text not null references jobs(id) on delete cascade,
+  customer_id text not null references customers(id) on delete cascade,
+  template_id text not null,
+  kind text not null,
+  storage_key text not null,
+  public_url text,
+  metadata_json text not null,
+  created_at text not null
+);
+
+create table if not exists billing_events (
+  id text primary key,
+  customer_id text not null references customers(id) on delete cascade,
+  job_id text not null references jobs(id) on delete cascade,
+  template_id text not null,
+  type text not null,
+  cost_usd real not null,
+  charge_usd real not null,
+  metadata_json text not null,
+  created_at text not null
+);
+
+create table if not exists webhook_deliveries (
+  id text primary key,
+  job_id text not null references jobs(id) on delete cascade,
+  customer_id text not null references customers(id) on delete cascade,
+  event_type text not null,
+  destination_url text not null,
+  payload_json text not null,
+  status text not null,
+  attempt_count integer not null default 0,
+  next_attempt_at text not null,
+  last_error text,
+  created_at text not null,
+  updated_at text not null
+);
+`);
+function mapJob(row) {
+    return {
+        id: String(row.id),
+        templateId: String(row.template_id),
+        operationName: String(row.operation_name),
+        workflowName: String(row.workflow_name),
+        tracer: String(row.tracer),
+        status: row.status,
+        customerId: String(row.customer_id),
+        payload: parseJson(row.payload_json, {}),
+        result: parseJson(row.result_json, null),
+        error: parseJson(row.error_json, null),
+        progress: Number(row.progress),
+        webhookUrl: row.webhook_url ? String(row.webhook_url) : null,
+        parentJobId: row.parent_job_id ? String(row.parent_job_id) : null,
+        priority: Number(row.priority),
+        attemptCount: Number(row.attempt_count),
+        maxAttempts: Number(row.max_attempts),
+        runAfter: String(row.run_after),
+        createdAt: String(row.created_at),
+        updatedAt: String(row.updated_at),
+        startedAt: row.started_at ? String(row.started_at) : null,
+        completedAt: row.completed_at ? String(row.completed_at) : null
+    };
+}
+export const database = {
+    raw: db,
+    upsertCustomer(input) {
+        const timestamp = nowIso();
+        db.prepare(`
+      insert into customers (id, email, name, default_webhook_url, created_at, updated_at)
+      values (@id, @email, @name, @default_webhook_url, @created_at, @updated_at)
+      on conflict(id) do update set
+        email = excluded.email,
+        name = excluded.name,
+        default_webhook_url = excluded.default_webhook_url,
+        updated_at = excluded.updated_at
+    `).run({
+            id: input.id,
+            email: input.email,
+            name: input.name ?? null,
+            default_webhook_url: input.defaultWebhookUrl ?? null,
+            created_at: timestamp,
+            updated_at: timestamp
+        });
+        return this.getCustomerById(input.id);
+    },
+    getCustomerById(id) {
+        const row = db.prepare(`select * from customers where id = ?`).get(id);
+        if (!row) {
+            return null;
+        }
+        return {
+            id: String(row.id),
+            email: String(row.email),
+            name: row.name ? String(row.name) : null,
+            defaultWebhookUrl: row.default_webhook_url ? String(row.default_webhook_url) : null
+        };
+    },
+    getCustomerByEmail(email) {
+        const row = db.prepare(`select * from customers where lower(email) = lower(?)`).get(email);
+        return row ? this.getCustomerById(String(row.id)) : null;
+    },
+    listProviderKeys(customerId) {
+        return db.prepare(`
+      select id, provider, label, status, weight, last_used_at, cooldown_until, disabled_reason, created_at, updated_at
+      from customer_provider_keys
+      where customer_id = ?
+      order by provider asc, created_at desc
+    `).all(customerId);
+    },
+    insertOtpChallenge(record) {
+        db.prepare(`
+      insert into otp_challenges (id, email, code_hash, expires_at, created_at)
+      values (@id, @email, @code_hash, @expires_at, @created_at)
+    `).run({
+            id: record.id,
+            email: record.email,
+            code_hash: record.codeHash,
+            expires_at: record.expiresAt,
+            created_at: nowIso()
+        });
+    },
+    findLatestOtpChallenge(email) {
+        return db.prepare(`
+      select * from otp_challenges
+      where lower(email) = lower(?)
+      order by created_at desc
+      limit 1
+    `).get(email);
+    },
+    consumeOtpChallenge(id) {
+        db.prepare(`update otp_challenges set consumed_at = ? where id = ?`).run(nowIso(), id);
+    },
+    insertApiKey(record) {
+        const timestamp = nowIso();
+        db.prepare(`
+      insert into api_keys (id, customer_id, key_hash, label, status, created_at, updated_at)
+      values (@id, @customer_id, @key_hash, @label, 'active', @created_at, @updated_at)
+    `).run({
+            id: record.id,
+            customer_id: record.customerId,
+            key_hash: record.keyHash,
+            label: record.label,
+            created_at: timestamp,
+            updated_at: timestamp
+        });
+    },
+    findApiKeyByHash(keyHash) {
+        return db.prepare(`
+      select a.*, c.email, c.name, c.default_webhook_url
+      from api_keys a
+      join customers c on c.id = a.customer_id
+      where a.key_hash = ? and a.status = 'active'
+    `).get(keyHash);
+    },
+    touchApiKey(id) {
+        db.prepare(`update api_keys set last_used_at = ?, updated_at = ? where id = ?`).run(nowIso(), nowIso(), id);
+    },
+    upsertTemplateConfig(record) {
+        const timestamp = nowIso();
+        db.prepare(`
+      insert into template_configs (id, customer_id, template_id, config_json, created_at, updated_at)
+      values (@id, @customer_id, @template_id, @config_json, @created_at, @updated_at)
+      on conflict(customer_id, template_id) do update set
+        config_json = excluded.config_json,
+        updated_at = excluded.updated_at
+    `).run({
+            id: record.id,
+            customer_id: record.customerId,
+            template_id: record.templateId,
+            config_json: stringifyJson(record.config),
+            created_at: timestamp,
+            updated_at: timestamp
+        });
+    },
+    getTemplateConfig(customerId, templateId) {
+        const row = db.prepare(`
+      select config_json from template_configs
+      where customer_id = ? and template_id = ?
+    `).get(customerId, templateId);
+        return parseJson(row?.config_json, {});
+    },
+    createJob(record) {
+        const timestamp = nowIso();
+        db.prepare(`
+      insert into jobs (
+        id, template_id, operation_name, workflow_name, tracer, status, customer_id,
+        payload_json, result_json, error_json, progress, webhook_url, parent_job_id,
+        priority, attempt_count, max_attempts, run_after, created_at, updated_at, started_at, completed_at
+      )
+      values (
+        @id, @template_id, @operation_name, @workflow_name, @tracer, @status, @customer_id,
+        @payload_json, @result_json, @error_json, @progress, @webhook_url, @parent_job_id,
+        @priority, @attempt_count, @max_attempts, @run_after, @created_at, @updated_at, @started_at, @completed_at
+      )
+    `).run({
+            id: record.id,
+            template_id: record.templateId,
+            operation_name: record.operationName,
+            workflow_name: record.workflowName,
+            tracer: record.tracer,
+            status: record.status,
+            customer_id: record.customerId,
+            payload_json: stringifyJson(record.payload),
+            result_json: stringifyJson(record.result ?? null),
+            error_json: stringifyJson(record.error ?? null),
+            progress: record.progress,
+            webhook_url: record.webhookUrl,
+            parent_job_id: record.parentJobId,
+            priority: record.priority,
+            attempt_count: record.attemptCount,
+            max_attempts: record.maxAttempts,
+            run_after: record.runAfter,
+            created_at: timestamp,
+            updated_at: timestamp,
+            started_at: record.startedAt ?? null,
+            completed_at: record.completedAt ?? null
+        });
+        return this.getJob(record.id);
+    },
+    getJob(id) {
+        const row = db.prepare(`select * from jobs where id = ?`).get(id);
+        return row ? mapJob(row) : null;
+    },
+    listRunnableJobs(limit) {
+        const rows = db.prepare(`
+      select * from jobs
+      where status = 'queued' and run_after <= ?
+      order by priority asc, created_at asc
+      limit ?
+    `).all(nowIso(), limit);
+        return rows.map(mapJob);
+    },
+    markJobRunning(id) {
+        const timestamp = nowIso();
+        db.prepare(`
+      update jobs
+      set status = 'running', started_at = coalesce(started_at, ?), updated_at = ?, attempt_count = attempt_count + 1
+      where id = ?
+    `).run(timestamp, timestamp, id);
+    },
+    updateJobStatus(input) {
+        const current = this.getJob(input.id);
+        db.prepare(`
+      update jobs
+      set status = @status,
+          progress = @progress,
+          result_json = @result_json,
+          error_json = @error_json,
+          run_after = @run_after,
+          completed_at = @completed_at,
+          updated_at = @updated_at
+      where id = @id
+    `).run({
+            id: input.id,
+            status: input.status,
+            progress: input.progress ?? current?.progress ?? 0,
+            result_json: input.result === undefined ? stringifyJson(current?.result ?? null) : stringifyJson(input.result),
+            error_json: input.error === undefined ? stringifyJson(current?.error ?? null) : stringifyJson(input.error),
+            run_after: input.runAfter ?? current?.runAfter ?? nowIso(),
+            completed_at: input.completedAt ?? current?.completedAt ?? null,
+            updated_at: nowIso()
+        });
+    },
+    listJobsForCustomer(customerId, templateId) {
+        const rows = templateId
+            ? db.prepare(`select * from jobs where customer_id = ? and template_id = ? order by created_at desc limit 100`).all(customerId, templateId)
+            : db.prepare(`select * from jobs where customer_id = ? order by created_at desc limit 100`).all(customerId);
+        return rows.map(mapJob);
+    },
+    addJobEvent(event) {
+        db.prepare(`
+      insert into job_events (id, job_id, level, message, metadata_json, progress, artifact_key, created_at)
+      values (@id, @job_id, @level, @message, @metadata_json, @progress, @artifact_key, @created_at)
+    `).run({
+            id: event.id,
+            job_id: event.jobId,
+            level: event.level,
+            message: event.message,
+            metadata_json: stringifyJson(event.metadata),
+            progress: event.progress,
+            artifact_key: event.artifactKey,
+            created_at: nowIso()
+        });
+    },
+    getJobEvents(jobId, since, limit = 100) {
+        const rows = since
+            ? db.prepare(`
+          select * from job_events where job_id = ? and created_at > ? order by created_at asc limit ?
+        `).all(jobId, since, limit)
+            : db.prepare(`
+          select * from job_events where job_id = ? order by created_at asc limit ?
+        `).all(jobId, limit);
+        return rows.map((row) => ({
+            id: String(row.id),
+            jobId: String(row.job_id),
+            level: row.level,
+            message: String(row.message),
+            metadata: parseJson(row.metadata_json, {}),
+            progress: row.progress === null ? null : Number(row.progress),
+            artifactKey: row.artifact_key ? String(row.artifact_key) : null,
+            createdAt: String(row.created_at)
+        }));
+    },
+    insertArtifact(record) {
+        db.prepare(`
+      insert into artifacts (id, job_id, customer_id, template_id, kind, storage_key, public_url, metadata_json, created_at)
+      values (@id, @job_id, @customer_id, @template_id, @kind, @storage_key, @public_url, @metadata_json, @created_at)
+    `).run({
+            id: record.id,
+            job_id: record.jobId,
+            customer_id: record.customerId,
+            template_id: record.templateId,
+            kind: record.kind,
+            storage_key: record.storageKey,
+            public_url: record.publicUrl ?? null,
+            metadata_json: stringifyJson(record.metadata ?? {}),
+            created_at: nowIso()
+        });
+    },
+    insertBillingEvent(record) {
+        db.prepare(`
+      insert into billing_events (id, customer_id, job_id, template_id, type, cost_usd, charge_usd, metadata_json, created_at)
+      values (@id, @customer_id, @job_id, @template_id, @type, @cost_usd, @charge_usd, @metadata_json, @created_at)
+    `).run({
+            id: record.id,
+            customer_id: record.customerId,
+            job_id: record.jobId,
+            template_id: record.templateId,
+            type: record.type,
+            cost_usd: record.costUsd,
+            charge_usd: record.chargeUsd,
+            metadata_json: stringifyJson(record.metadata ?? {}),
+            created_at: nowIso()
+        });
+    },
+    queueWebhookDelivery(record) {
+        const timestamp = nowIso();
+        db.prepare(`
+      insert into webhook_deliveries (
+        id, job_id, customer_id, event_type, destination_url, payload_json, status,
+        attempt_count, next_attempt_at, created_at, updated_at
+      ) values (
+        @id, @job_id, @customer_id, @event_type, @destination_url, @payload_json, 'pending',
+        0, @next_attempt_at, @created_at, @updated_at
+      )
+    `).run({
+            id: record.id,
+            job_id: record.jobId,
+            customer_id: record.customerId,
+            event_type: record.eventType,
+            destination_url: record.destinationUrl,
+            payload_json: stringifyJson(record.payload),
+            next_attempt_at: timestamp,
+            created_at: timestamp,
+            updated_at: timestamp
+        });
+    },
+    listPendingWebhookDeliveries(limit) {
+        return db.prepare(`
+      select * from webhook_deliveries
+      where status in ('pending', 'retrying') and next_attempt_at <= ?
+      order by next_attempt_at asc
+      limit ?
+    `).all(nowIso(), limit);
+    },
+    markWebhookDelivery(record) {
+        db.prepare(`
+      update webhook_deliveries
+      set status = @status,
+          attempt_count = @attempt_count,
+          next_attempt_at = coalesce(@next_attempt_at, next_attempt_at),
+          last_error = @last_error,
+          updated_at = @updated_at
+      where id = @id
+    `).run({
+            id: record.id,
+            status: record.status,
+            attempt_count: record.attemptCount,
+            next_attempt_at: record.nextAttemptAt ?? null,
+            last_error: record.lastError ?? null,
+            updated_at: nowIso()
+        });
+    },
+    acquireProviderKeyLease(input) {
+        const transaction = db.transaction(() => {
+            db.prepare(`delete from provider_key_leases where expires_at <= ?`).run(nowIso());
+            const row = db.prepare(`
+        select k.id, k.encrypted_secret
+        from customer_provider_keys k
+        left join provider_key_leases l on l.key_id = k.id and l.expires_at > ?
+        where k.customer_id = ?
+          and k.provider = ?
+          and k.status = 'active'
+          and (k.cooldown_until is null or k.cooldown_until <= ?)
+          and l.key_id is null
+        order by case when k.last_used_at is null then 0 else 1 end asc, k.last_used_at asc, k.weight desc
+        limit 1
+      `).get(nowIso(), input.customerId, input.provider, nowIso());
+            if (!row) {
+                return null;
+            }
+            db.prepare(`
+        insert into provider_key_leases (key_id, lease_token, worker_id, job_id, leased_at, expires_at)
+        values (?, ?, ?, ?, ?, ?)
+      `).run(row.id, input.leaseToken, input.workerId, input.jobId, nowIso(), input.expiresAt);
+            return { keyId: row.id, encryptedSecret: row.encrypted_secret };
+        });
+        return transaction();
+    },
+    releaseProviderKeyLease(input) {
+        db.prepare(`delete from provider_key_leases where key_id = ? and lease_token = ?`).run(input.keyId, input.leaseToken);
+    },
+    recordProviderKeyUsage(input) {
+        db.prepare(`
+      insert into provider_key_usage_events (
+        id, key_id, job_id, provider, model, event_type, input_tokens, output_tokens, cost_usd, metadata, created_at
+      ) values (
+        @id, @key_id, @job_id, @provider, @model, @event_type, @input_tokens, @output_tokens, @cost_usd, @metadata, @created_at
+      )
+    `).run({
+            id: input.id,
+            key_id: input.keyId,
+            job_id: input.jobId,
+            provider: input.provider,
+            model: input.model ?? null,
+            event_type: input.eventType,
+            input_tokens: input.inputTokens ?? null,
+            output_tokens: input.outputTokens ?? null,
+            cost_usd: input.costUsd ?? null,
+            metadata: stringifyJson(input.metadata ?? {}),
+            created_at: nowIso()
+        });
+    },
+    touchProviderKey(keyId) {
+        db.prepare(`update customer_provider_keys set last_used_at = ?, updated_at = ? where id = ?`).run(nowIso(), nowIso(), keyId);
+    },
+    setProviderKeyCooldown(keyId, cooldownUntil, status = "active", disabledReason = null) {
+        db.prepare(`
+      update customer_provider_keys
+      set cooldown_until = ?, status = ?, disabled_reason = ?, updated_at = ?
+      where id = ?
+    `).run(cooldownUntil, status, disabledReason, nowIso(), keyId);
+    },
+    createProviderKey(record) {
+        const timestamp = nowIso();
+        db.prepare(`
+      insert into customer_provider_keys (
+        id, customer_id, provider, label, encrypted_secret, status, weight, created_at, updated_at
+      ) values (
+        @id, @customer_id, @provider, @label, @encrypted_secret, 'active', @weight, @created_at, @updated_at
+      )
+    `).run({
+            id: record.id,
+            customer_id: record.customerId,
+            provider: record.provider,
+            label: record.label,
+            encrypted_secret: record.encryptedSecret,
+            weight: record.weight,
+            created_at: timestamp,
+            updated_at: timestamp
+        });
+    }
+};