@kelpie/server 0.1.0

package/src/modules/import-export/routes.ts

@@ -0,0 +1,276 @@
+import {
+  IMPORT_CONFLICT_MODES,
+  IMPORT_OBJECTS,
+  IMPORT_SOURCES,
+  MAX_IMPORT_FILE_BYTES,
+  defaultMatchKeyId,
+} from '@kelpie/schemas'
+import type { ImportColumnMap, ImportObject } from '@kelpie/schemas'
+import type { Context, Hono } from 'hono'
+import { stream } from 'hono/streaming'
+import { z } from 'zod'
+
+import { AppError, toErrorDetails } from '../../lib/errors.ts'
+import type { Actor } from '../auth/actor.ts'
+import { resolveActorFrom } from '../auth/credentials.ts'
+import type { CredentialDependencies } from '../auth/credentials.ts'
+import type { ImportExportService, ImportJobView } from './service.ts'
+
+/**
+ * `/v1/export` and `/v1/import`, per `import-export.md`.
+ *
+ * The upload is `multipart/form-data` rather than JSON, because the thing being
+ * uploaded is a file and base64 in a JSON body would put a third of the ten
+ * megabyte limit into encoding. Its text fields are validated by the same Zod
+ * schema a JSON body would use.
+ */
+
+/** The file name a download is offered under. */
+const DOWNLOAD_PREFIX = 'kelpie'
+
+const objectFile = z.enum(IMPORT_OBJECTS.map((object) => `${object}.csv`) as [string, ...string[]])
+
+/**
+ * `column_map` arrives as a JSON string inside the form, since a multipart field
+ * carries text. Absent means "derive one", which is not the same as `{}`: an
+ * empty object is a map that ignores every column, and it fails the required
+ * column check as it should.
+ */
+const columnMapSchema = z.record(z.string(), z.string().nullable())
+
+const createJobForm = z.object({
+  source: z.enum(IMPORT_SOURCES),
+  object: z.enum(IMPORT_OBJECTS),
+  conflict_mode: z.enum(IMPORT_CONFLICT_MODES).default('skip'),
+  match_key: z.string().min(1).optional(),
+  column_map: z.string().optional(),
+  dry_run: z.enum(['true', 'false']).default('true'),
+})
+
+export interface ImportExportRoutesDependencies extends CredentialDependencies {
+  readonly service: ImportExportService
+}
+
+export function importJobResponse(job: ImportJobView): Record<string, unknown> {
+  return {
+    id: job.id,
+    source: job.source,
+    object: job.object,
+    status: job.status,
+    conflict_mode: job.conflictMode,
+    match_key: job.matchKey,
+    column_map: job.columnMap,
+    source_headers: job.sourceHeaders,
+    file_name: job.fileName,
+    counts: job.counts,
+    errors: job.errors,
+    preview: job.preview,
+    created_at: job.createdAt.toISOString(),
+    updated_at: job.updatedAt.toISOString(),
+  }
+}
+
+/** `people.csv` → `people`. The route's own param, so an unknown one is a 404. */
+function readObjectFile(context: Context): ImportObject {
+  const parsed = objectFile.safeParse(context.req.param('file'))
+
+  if (!parsed.success) {
+    throw AppError.notFound(
+      `Export ${IMPORT_OBJECTS.map((object) => `${object}.csv`).join(', ')}`,
+    )
+  }
+
+  return parsed.data.replace(/\.csv$/u, '') as ImportObject
+}
+
+function csvHeaders(fileName: string): Record<string, string> {
+  return {
+    'Content-Type': 'text/csv; charset=utf-8',
+    'Content-Disposition': `attachment; filename="${fileName}"`,
+    // A workspace's data is not a thing to leave in a shared cache.
+    'Cache-Control': 'private, no-store',
+  }
+}
+
+function readColumnMap(raw: string | undefined): ImportColumnMap | undefined {
+  if (raw === undefined) {
+    return undefined
+  }
+
+  const parsed: unknown = ((): unknown => {
+    try {
+      return JSON.parse(raw)
+    } catch {
+      throw AppError.validationFailed('column_map must be a JSON object', [
+        { field: 'column_map', message: 'Could not read it as JSON' },
+      ])
+    }
+  })()
+
+  const map = columnMapSchema.safeParse(parsed)
+
+  if (!map.success) {
+    throw AppError.validationFailed(
+      'column_map must map Kelpie columns to source headers or null',
+      toErrorDetails(map.error.issues),
+    )
+  }
+
+  return map.data
+}
+
+/**
+ * Pulls the upload off the request.
+ *
+ * @throws AppError 400 when the body is not multipart at all, 422 when the file
+ *   is missing or over the limit. A body Hono cannot read as a form is the
+ *   caller sending the wrong content type, which is a different mistake from
+ *   sending a form with the wrong fields in it.
+ */
+async function readUpload(context: Context): Promise<{ file: File; fields: Record<string, string> }> {
+  const body = await context.req.parseBody().catch(() => {
+    throw new AppError('bad_request', 'Send the job as multipart/form-data')
+  })
+  const file = body.file
+
+  if (!(file instanceof File)) {
+    throw AppError.validationFailed('The job needs a CSV file', [
+      { field: 'file', message: 'Missing' },
+    ])
+  }
+
+  if (file.size > MAX_IMPORT_FILE_BYTES) {
+    throw AppError.validationFailed(
+      `A file may be at most ${String(MAX_IMPORT_FILE_BYTES / 1024 / 1024)} MB`,
+      [{ field: 'file', message: `This one is ${String(file.size)} bytes` }],
+    )
+  }
+
+  const fields: Record<string, string> = {}
+
+  for (const [key, value] of Object.entries(body)) {
+    if (typeof value === 'string') {
+      fields[key] = value
+    }
+  }
+
+  return { file, fields }
+}
+
+export function mountImportExportRoutes(
+  router: Hono,
+  dependencies: ImportExportRoutesDependencies,
+): void {
+  const requireActor = (context: Context): Promise<Actor> => resolveActorFrom(dependencies, context)
+
+  /**
+   * Streams the object as CSV.
+   *
+   * The actor is resolved before the stream opens. A `401` has to be a `401`
+   * with an error body, and once the first chunk is written the status is
+   * already sent.
+   */
+  router.get('/export/:file', async (context) => {
+    const object = readObjectFile(context)
+    const actor = await requireActor(context)
+    const lines = dependencies.service.exportCsv(actor, object)
+    const encoder = new TextEncoder()
+
+    for (const [header, value] of Object.entries(
+      csvHeaders(`${DOWNLOAD_PREFIX}-${object}.csv`),
+    )) {
+      context.header(header, value)
+    }
+
+    return stream(context, async (writable) => {
+      for await (const line of lines) {
+        await writable.write(encoder.encode(line))
+      }
+    })
+  })
+
+  router.get('/export/templates/:file', async (context) => {
+    const object = readObjectFile(context)
+
+    await requireActor(context)
+
+    return context.body(
+      dependencies.service.templateCsv(object),
+      200,
+      csvHeaders(`${DOWNLOAD_PREFIX}-${object}-template.csv`),
+    )
+  })
+
+  /**
+   * Creates a job and dry-runs it.
+   *
+   * `dry_run` must be `true`: `import-export.md` makes the commit a separate
+   * call, so a create claiming otherwise is asking for something this endpoint
+   * will not do rather than something it can quietly ignore.
+   */
+  router.post('/import/jobs', async (context) => {
+    const actor = await requireActor(context)
+    const { file, fields } = await readUpload(context)
+    const parsed = createJobForm.safeParse(fields)
+
+    if (!parsed.success) {
+      throw AppError.validationFailed('The job is missing something', toErrorDetails(parsed.error.issues))
+    }
+
+    if (parsed.data.dry_run !== 'true') {
+      throw AppError.validationFailed('A new job is always a dry run', [
+        {
+          field: 'dry_run',
+          message: 'Create the job, then POST to /import/jobs/{id}/commit',
+        },
+      ])
+    }
+
+    const job = await dependencies.service.createJob(actor, {
+      source: parsed.data.source,
+      object: parsed.data.object,
+      conflictMode: parsed.data.conflict_mode,
+      matchKeyId: parsed.data.match_key ?? defaultMatchKeyId(parsed.data.object),
+      columnMap: readColumnMap(parsed.data.column_map),
+      fileName: file.name.length === 0 ? null : file.name,
+      csv: await file.text(),
+    })
+
+    // 202 when the work is still running, 201 when it is done. Both created the
+    // job; only one of them has an answer about the file in it yet.
+    return context.json(importJobResponse(job), job.status === 'validating' ? 202 : 201)
+  })
+
+  router.get('/import/jobs/:id', async (context) => {
+    const job = await dependencies.service.getJob(
+      await requireActor(context),
+      context.req.param('id'),
+    )
+
+    return context.json(importJobResponse(job))
+  })
+
+  router.delete('/import/jobs/:id', async (context) => {
+    await dependencies.service.deleteJob(await requireActor(context), context.req.param('id'))
+
+    return context.body(null, 204)
+  })
+
+  /**
+   * Commits a job, applying the file sent with it.
+   *
+   * Multipart like the create, and for the same reason: a job keeps only the
+   * digest of the file it forecast, so the bytes come back here rather than
+   * sitting in a column between the two calls.
+   */
+  router.post('/import/jobs/:id/commit', async (context) => {
+    const { file } = await readUpload(context)
+    const job = await dependencies.service.commit(
+      await requireActor(context),
+      context.req.param('id'),
+      await file.text(),
+    )
+
+    return context.json(importJobResponse(job), job.status === 'committing' ? 202 : 200)
+  })
+}

package/src/modules/import-export/schema.ts

@@ -0,0 +1,142 @@
+import {
+  IMPORT_CONFLICT_MODES,
+  IMPORT_JOB_STATUSES,
+  IMPORT_OBJECTS,
+  IMPORT_ROW_ACTIONS,
+  IMPORT_SOURCES,
+} from '@kelpie/schemas'
+import type {
+  ImportColumnMap,
+  ImportCounts,
+  ImportPreviewRow,
+  ImportRowError,
+} from '@kelpie/schemas'
+import { index, integer, jsonb, pgTable, primaryKey, text } from 'drizzle-orm/pg-core'
+
+import { checkOneOf, createdAt, primaryId, updatedAt } from '../../lib/columns.ts'
+import { workspaces } from '../workspace/schema.ts'
+
+/**
+ * The fixed value sets come from `@kelpie/schemas`, so these check constraints,
+ * the route's Zod enums, and the browser's decoder are one list rather than
+ * three copies.
+ */
+export {
+  IMPORT_CONFLICT_MODES,
+  IMPORT_JOB_STATUSES,
+  IMPORT_OBJECTS,
+  IMPORT_ROW_ACTIONS,
+  IMPORT_SOURCES,
+} from '@kelpie/schemas'
+export type {
+  ImportConflictMode,
+  ImportJobStatus,
+  ImportObject,
+  ImportRowAction,
+  ImportSource,
+  SettledRowAction,
+} from '@kelpie/schemas'
+
+/** One problem with one cell, as stored against the row that carries it. */
+export interface StoredRowError {
+  readonly field: string
+  readonly message: string
+}
+
+/**
+ * A CSV import, from upload through dry run to commit.
+ *
+ * `column_map`, `counts`, `errors` and `preview` are jsonb because their shape
+ * follows the object being imported and nothing queries into them.
+ *
+ * A job stores no part of the file. The upload plans it in memory and keeps
+ * `counts`, `errors`, `preview` and `file_sha256`; the caller keeps the file and
+ * hands it back at commit. That is what makes an abandoned dry run cost one row
+ * rather than a copy of a ten megabyte upload, and it is why nothing is exploded
+ * into `import_job_rows` until an import actually runs.
+ *
+ * `file_name` is what the uploader called the file, for the mapping screen.
+ */
+export const importJobs = pgTable(
+  'import_jobs',
+  {
+    id: primaryId(),
+    workspaceId: text('workspace_id')
+      .notNull()
+      .references(() => workspaces.id, { onDelete: 'cascade' }),
+    source: text('source').notNull(),
+    object: text('object').notNull(),
+    status: text('status').notNull(),
+    conflictMode: text('conflict_mode').notNull(),
+    matchKey: text('match_key').notNull(),
+    columnMap: jsonb('column_map').$type<ImportColumnMap>().notNull().default({}),
+    sourceHeaders: jsonb('source_headers').$type<readonly string[]>().notNull().default([]),
+    counts: jsonb('counts').$type<ImportCounts>().notNull(),
+    /** The first `IMPORT_REPORTED_ERRORS` failing rows. `counts.error` is the true number. */
+    errors: jsonb('errors').$type<readonly ImportRowError[]>().notNull().default([]),
+    /** The first `IMPORT_PREVIEW_ROWS` rows, mapped as Kelpie read them. */
+    preview: jsonb('preview').$type<readonly ImportPreviewRow[]>().notNull().default([]),
+    /**
+     * SHA-256 of the file this job forecast, hex.
+     *
+     * The file itself is not kept. The commit carries it back and is refused if
+     * it hashes to anything else, which buys the same "this is the file you
+     * approved" guarantee as storing it for 64 characters instead of up to ten
+     * megabytes. Null only on a job from before this, which cannot be committed;
+     * the remedy is the one `import-export.md` already gives — upload it again.
+     */
+    fileSha256: text('file_sha256'),
+    fileName: text('file_name'),
+    /** Why a background pass gave up. Null unless `status` is `failed`. */
+    failureReason: text('failure_reason'),
+    createdAt: createdAt(),
+    updatedAt: updatedAt(),
+  },
+  (table) => [
+    index('import_jobs_workspace_idx').on(table.workspaceId),
+    checkOneOf('import_jobs_source_check', table.source, IMPORT_SOURCES),
+    checkOneOf('import_jobs_object_check', table.object, IMPORT_OBJECTS),
+    checkOneOf('import_jobs_conflict_mode_check', table.conflictMode, IMPORT_CONFLICT_MODES),
+    checkOneOf('import_jobs_status_check', table.status, IMPORT_JOB_STATUSES),
+  ],
+)
+
+/**
+ * What an import did to one line of the file.
+ *
+ * Written by the commit, one row at a time, in the same transaction as the
+ * record that row wrote. Nothing here exists before then: a dry run forecasts
+ * without storing, so a caller who corrects a mapping four times leaves four
+ * jobs and no rows rather than four copies of the file.
+ *
+ * `action` is therefore always what happened, never a forecast. `values` holds
+ * the line as it arrived — **source header → cell** — so the row says what it
+ * acted on without the reader having to go back to `import_jobs.csv` and count
+ * lines.
+ */
+export const importJobRows = pgTable(
+  'import_job_rows',
+  {
+    workspaceId: text('workspace_id')
+      .notNull()
+      .references(() => workspaces.id, { onDelete: 'cascade' }),
+    jobId: text('job_id')
+      .notNull()
+      .references(() => importJobs.id, { onDelete: 'cascade' }),
+    /** The line in the file, counting the header as line 1. Data starts at 2. */
+    rowNumber: integer('row_number').notNull(),
+    values: jsonb('values').$type<Readonly<Record<string, string>>>().notNull().default({}),
+    action: text('action').notNull(),
+    errors: jsonb('errors').$type<readonly StoredRowError[]>().notNull().default([]),
+    createdAt: createdAt(),
+    updatedAt: updatedAt(),
+  },
+  (table) => [
+    // The job and the line already identify a row, so it needs no id of its own.
+    // A `<prefix>_<ulid>` here would also mean a prefix `api.md` does not
+    // document, for something that never appears on the wire.
+    primaryKey({ columns: [table.jobId, table.rowNumber] }),
+    index('import_job_rows_workspace_idx').on(table.workspaceId),
+    checkOneOf('import_job_rows_action_check', table.action, IMPORT_ROW_ACTIONS),
+  ],
+)