@kelpie/server 0.1.0

package/src/modules/import-export/streams.ts

@@ -0,0 +1,98 @@
+import type { ImportObject } from '@kelpie/schemas'
+
+import type { Database } from '../../lib/database.ts'
+import { csvLine } from './csv.ts'
+import { companyCells, dealCells, headersFor, personCells, positionCells } from './exportRows.ts'
+import * as repository from './repository.ts'
+
+/**
+ * A whole object as CSV, a page of records at a time.
+ *
+ * A generator rather than a string, because a workspace's deals do not have to
+ * fit in memory to be downloaded and `import-export.md` says the endpoint
+ * streams. The route pipes this straight into the response.
+ *
+ * Paging is a keyset on `id`. Ids are ULIDs, so that is creation order, and
+ * unlike an offset it cannot skip or repeat a record when one is written while
+ * the download is in flight.
+ */
+
+/** Reads one page and reports where the next starts. */
+interface Pager<TRow extends { readonly id: string }> {
+  read(after: string): Promise<TRow[]>
+  cells(rows: readonly TRow[]): Promise<readonly (readonly string[])[]>
+}
+
+async function* pageThrough<TRow extends { readonly id: string }>(
+  pager: Pager<TRow>,
+): AsyncGenerator<string> {
+  let after = ''
+
+  for (;;) {
+    const rows = await pager.read(after)
+
+    if (rows.length === 0) {
+      return
+    }
+
+    for (const cells of await pager.cells(rows)) {
+      yield csvLine(cells)
+    }
+
+    const last = rows.at(-1)
+
+    if (last === undefined || rows.length < repository.EXPORT_PAGE) {
+      return
+    }
+
+    after = last.id
+  }
+}
+
+/** The rows of one page, all their people in one query rather than one per deal. */
+async function dealPage(
+  db: Database,
+  rows: readonly repository.ExportDealRow[],
+): Promise<readonly (readonly string[])[]> {
+  const emails = await repository.readDealPersonEmails(
+    db,
+    rows.map((row) => row.id),
+  )
+
+  return rows.map((row) => dealCells(row, emails.get(row.id) ?? []))
+}
+
+export async function* streamExport(
+  db: Database,
+  workspaceId: string,
+  object: ImportObject,
+): AsyncGenerator<string> {
+  yield csvLine(headersFor(object))
+
+  switch (object) {
+    case 'companies':
+      yield* pageThrough({
+        read: (after) => repository.readCompanies(db, workspaceId, after),
+        cells: (rows) => Promise.resolve(rows.map(companyCells)),
+      })
+      return
+    case 'people':
+      yield* pageThrough({
+        read: (after) => repository.readPeople(db, workspaceId, after),
+        cells: (rows) => Promise.resolve(rows.map(personCells)),
+      })
+      return
+    case 'positions':
+      yield* pageThrough({
+        read: (after) => repository.readPositions(db, workspaceId, after),
+        cells: (rows) => Promise.resolve(rows.map(positionCells)),
+      })
+      return
+    case 'deals':
+      yield* pageThrough({
+        read: (after) => repository.readDeals(db, workspaceId, after),
+        cells: (rows) => dealPage(db, rows),
+      })
+      return
+  }
+}

package/src/modules/import-export/tools.ts

@@ -0,0 +1,153 @@
+import {
+  IMPORT_CONFLICT_MODES,
+  IMPORT_OBJECTS,
+  IMPORT_SOURCES,
+  MAX_IMPORT_FILE_BYTES,
+  defaultMatchKeyId,
+} from '@kelpie/schemas'
+import { z } from 'zod'
+
+import { AppError } from '../../lib/errors.ts'
+import type { McpToolRegistry } from '../../runtime/module.ts'
+import { idArg } from '../crudTools.ts'
+import { importJobResponse } from './routes.ts'
+import type { ImportExportService } from './service.ts'
+
+/**
+ * The four tools `import-export.md` names: `export_csv`, `import_preview`,
+ * `import_commit`, `import_job_get`.
+ *
+ * The file arrives as a string rather than as multipart, which is the one
+ * difference from the REST endpoints. Both surfaces call the same service with
+ * the same bytes; multipart exists on the wire because a browser uploads a file,
+ * and MCP has no file to upload.
+ */
+
+/**
+ * How much CSV `export_csv` will inline.
+ *
+ * A tool result is read into a model's context, so a whole workspace's people is
+ * the wrong thing to hand back by accident. Over the limit the tool refuses and
+ * names the REST download, rather than truncating: half a CSV looks exactly like
+ * a whole one, and an agent would draw conclusions from the missing rows.
+ */
+export const MAX_INLINE_EXPORT_BYTES = 256 * 1024
+
+const objectArg = z
+  .enum(IMPORT_OBJECTS)
+  .describe(`Which object to move. One of: ${IMPORT_OBJECTS.join(', ')}.`)
+
+/** `column_map` is a real object here. The multipart route takes it as a JSON string. */
+const columnMapArg = z
+  .record(z.string(), z.string().nullable())
+  .describe(
+    'Kelpie column to source header, or null to ignore that column. Omit to derive one ' +
+      'from the source preset and the file\'s own headers.',
+  )
+
+const csvArg = z
+  .string()
+  .min(1)
+  .max(MAX_IMPORT_FILE_BYTES)
+  .describe('The CSV file, as text. UTF-8, with a header row.')
+
+const previewArgs = z.strictObject({
+  source: z.enum(IMPORT_SOURCES).describe('Which export the file came out of.'),
+  object: objectArg,
+  csv: csvArg,
+  conflict_mode: z
+    .enum(IMPORT_CONFLICT_MODES)
+    .default('skip')
+    .describe('What to do with a row that matches an existing record.'),
+  match_key: z.string().min(1).optional().describe('Which field decides that match.'),
+  column_map: columnMapArg.optional(),
+  file_name: z.string().min(1).nullable().default(null).describe('Recorded on the job, for the log.'),
+})
+
+async function collectCsv(lines: AsyncIterable<string>): Promise<string> {
+  const parts: string[] = []
+  let bytes = 0
+
+  for await (const line of lines) {
+    bytes += Buffer.byteLength(line, 'utf8')
+
+    if (bytes > MAX_INLINE_EXPORT_BYTES) {
+      throw AppError.validationFailed(
+        `That export is over ${String(MAX_INLINE_EXPORT_BYTES / 1024)} KB, which is more than a tool result should carry`,
+        [
+          {
+            field: 'object',
+            message: 'Download it from GET /v1/export/{object}.csv with the same API key',
+          },
+        ],
+      )
+    }
+
+    parts.push(line)
+  }
+
+  return parts.join('')
+}
+
+export function registerImportExportTools(mcp: McpToolRegistry, service: ImportExportService): void {
+  mcp.tool({
+    name: 'export_csv',
+    description:
+      'Export every record of one object as CSV text. Refuses an export too large to sit in ' +
+      'a tool result, and names the download to use instead. Mirrors GET /v1/export/{object}.csv.',
+    inputSchema: z.strictObject({ object: objectArg }),
+    invoke: async ({ object }, actor) => ({
+      object,
+      csv: await collectCsv(service.exportCsv(actor, object)),
+    }),
+  })
+
+  mcp.tool({
+    name: 'export_template_csv',
+    description:
+      'The header row Kelpie expects for one object, with no records in it. Start a hand-built ' +
+      'import from this. Mirrors GET /v1/export/templates/{object}.csv.',
+    inputSchema: z.strictObject({ object: objectArg }),
+    invoke: ({ object }) => Promise.resolve({ object, csv: service.templateCsv(object) }),
+  })
+
+  mcp.tool({
+    name: 'import_preview',
+    description:
+      'Dry-run a CSV: creates an import job, reports per-row create, update, skip and error ' +
+      'counts, and writes nothing. Commit it afterwards with import_commit and the same file. ' +
+      'Mirrors POST /v1/import/jobs.',
+    inputSchema: previewArgs,
+    invoke: async (args, actor) =>
+      importJobResponse(
+        await service.createJob(actor, {
+          source: args.source,
+          object: args.object,
+          conflictMode: args.conflict_mode,
+          matchKeyId: args.match_key ?? defaultMatchKeyId(args.object),
+          columnMap: args.column_map,
+          fileName: args.file_name,
+          csv: args.csv,
+        }),
+      ),
+  })
+
+  mcp.tool({
+    name: 'import_commit',
+    description:
+      'Apply a job that was dry-run. Send back the same file: a job keeps only its digest, ' +
+      'and a different one is refused. Re-running a commit is safe. Mirrors ' +
+      'POST /v1/import/jobs/{id}/commit.',
+    inputSchema: z.strictObject({ id: idArg, csv: csvArg }),
+    invoke: async ({ id, csv }, actor) => importJobResponse(await service.commit(actor, id, csv)),
+  })
+
+  mcp.tool({
+    name: 'import_job_get',
+    description:
+      'Read a job: status, counts and per-row errors. Poll this while a job is validating or ' +
+      'committing. Mirrors GET /v1/import/jobs/{id}.',
+    inputSchema: z.strictObject({ id: idArg }),
+    invoke: async ({ id }, actor) => importJobResponse(await service.getJob(actor, id)),
+  })
+}

package/src/modules/import-export/validation.ts

@@ -0,0 +1,163 @@
+import {
+  ACCOUNT_TYPES,
+  COMPANY_STAGES,
+  ICP_FITS,
+  INFLUENCE_LEVELS,
+  PREFERRED_CHANNELS,
+  RELATIONSHIP_LEVELS,
+  SIZE_BANDS,
+  requiredColumns,
+} from '@kelpie/schemas'
+import type { ImportObject, MatchKeyOption } from '@kelpie/schemas'
+
+import { isoDateSchema } from '../../lib/dates.ts'
+import type { StoredRowError } from './schema.ts'
+import { moneyToCents } from './mapping.ts'
+
+/**
+ * What a row must look like before anything is looked up.
+ *
+ * Shape only: required cells present, enums recognised, numbers numeric, dates
+ * real. Whether the company a deal names exists is resolution, not validation,
+ * and lives in `plan.ts` where the workspace is in hand.
+ *
+ * Every problem with a row is reported at once. A row fixed one message at a
+ * time is four upload cycles, and the caller has the whole row in front of them.
+ *
+ * Pure.
+ */
+
+/** Something on both sides of an `@` and no whitespace. */
+const EMAIL_SHAPE = /^[^\s@]+@[^\s@]+$/u
+
+/**
+ * A fixed value set, matched case-insensitively.
+ *
+ * `import-export.md` says an unknown enum is a row error rather than silently
+ * coerced, and that still holds: `Prospect` and `prospect` are the same value
+ * written two ways, which is what a person exports from a spreadsheet, while
+ * `Client` is unknown and fails.
+ */
+function readEnum(values: readonly string[], raw: string): string | undefined {
+  return values.find((value) => value.toLowerCase() === raw.trim().toLowerCase())
+}
+
+/** @returns The canonical stored value, or an error naming what was refused. */
+function checkEnum(
+  errors: StoredRowError[],
+  field: string,
+  values: readonly string[],
+  raw: string | undefined,
+): void {
+  if (raw === undefined || raw.trim().length === 0) {
+    return
+  }
+
+  if (readEnum(values, raw) === undefined) {
+    errors.push({
+      field,
+      message: `Unknown ${field} "${raw.trim()}". Use one of: ${values.join(', ')}`,
+    })
+  }
+}
+
+/** The canonical form of an enum cell, or null when the cell is blank. */
+export function canonicalEnum(values: readonly string[], raw: string | undefined): string | null {
+  if (raw === undefined || raw.trim().length === 0) {
+    return null
+  }
+
+  return readEnum(values, raw) ?? null
+}
+
+function checkRequired(
+  errors: StoredRowError[],
+  object: ImportObject,
+  matchKey: MatchKeyOption,
+  mapped: Readonly<Record<string, string>>,
+): void {
+  for (const column of requiredColumns(object, matchKey)) {
+    if ((mapped[column] ?? '').trim().length === 0) {
+      errors.push({ field: column, message: 'Missing required field' })
+    }
+  }
+}
+
+function checkCompany(errors: StoredRowError[], mapped: Readonly<Record<string, string>>): void {
+  checkEnum(errors, 'stage', COMPANY_STAGES, mapped.stage)
+  checkEnum(errors, 'size_band', SIZE_BANDS, mapped.size_band)
+  checkEnum(errors, 'account_type', ACCOUNT_TYPES, mapped.account_type)
+  checkEnum(errors, 'icp_fit', ICP_FITS, mapped.icp_fit)
+}
+
+function checkPerson(errors: StoredRowError[], mapped: Readonly<Record<string, string>>): void {
+  const email = (mapped.email ?? '').trim()
+
+  if (email.length > 0 && !EMAIL_SHAPE.test(email)) {
+    errors.push({ field: 'email', message: `"${email}" is not an email address` })
+  }
+
+  checkEnum(errors, 'preferred_channel', PREFERRED_CHANNELS, mapped.preferred_channel)
+  checkEnum(errors, 'influence', INFLUENCE_LEVELS, mapped.influence)
+  checkEnum(errors, 'relationship', RELATIONSHIP_LEVELS, mapped.relationship)
+}
+
+function checkPosition(errors: StoredRowError[], mapped: Readonly<Record<string, string>>): void {
+  const email = (mapped.person_email ?? '').trim()
+
+  if (email.length > 0 && !EMAIL_SHAPE.test(email)) {
+    errors.push({ field: 'person_email', message: `"${email}" is not an email address` })
+  }
+}
+
+function checkDeal(errors: StoredRowError[], mapped: Readonly<Record<string, string>>): void {
+  if (moneyToCents(mapped.value) === undefined) {
+    errors.push({ field: 'value', message: `"${mapped.value ?? ''}" is not a number` })
+  }
+
+  const close = (mapped.expected_close ?? '').trim()
+
+  if (close.length > 0 && !isoDateSchema.safeParse(close).success) {
+    errors.push({ field: 'expected_close', message: `"${close}" is not a date. Use YYYY-MM-DD` })
+  }
+
+  const owner = (mapped.owner_email ?? '').trim()
+
+  if (owner.length > 0 && !EMAIL_SHAPE.test(owner)) {
+    errors.push({ field: 'owner_email', message: `"${owner}" is not an email address` })
+  }
+}
+
+/** @returns Every shape problem with the row, or an empty array when it is usable. */
+export function validateRow(
+  object: ImportObject,
+  matchKey: MatchKeyOption,
+  mapped: Readonly<Record<string, string>>,
+): readonly StoredRowError[] {
+  const errors: StoredRowError[] = []
+
+  checkRequired(errors, object, matchKey, mapped)
+
+  // Past a missing required cell the rest is noise: an absent email is not also
+  // a malformed one, and the caller has one thing to fix either way.
+  if (errors.length > 0) {
+    return errors
+  }
+
+  switch (object) {
+    case 'companies':
+      checkCompany(errors, mapped)
+      break
+    case 'people':
+      checkPerson(errors, mapped)
+      break
+    case 'positions':
+      checkPosition(errors, mapped)
+      break
+    case 'deals':
+      checkDeal(errors, mapped)
+      break
+  }
+
+  return errors
+}