@kelpie/server 0.1.0

package/src/modules/import-export/csv.ts

@@ -0,0 +1,180 @@
+import { createHash } from 'node:crypto'
+
+/**
+ * RFC 4180 CSV, read and written.
+ *
+ * Ported from `mockups/src/data/importExport.ts` with two corrections. A blank
+ * line no longer shifts the numbering of every row after it, because a row
+ * number is what an error message points the user at. And a file whose header
+ * row repeats a name is refused rather than silently read as its last
+ * occurrence, because a `column_map` names a header and there would be no way to
+ * say which of the two it meant.
+ *
+ * Pure: no I/O, no clock. The whole of it is unit-tested.
+ */
+
+/**
+ * The digest a job keeps instead of the file.
+ *
+ * A dry run records this, and the commit that carries the file back is checked
+ * against it. Storing 64 characters rather than up to ten megabytes is the
+ * point: a forecast nobody committed should not cost a workspace the file.
+ *
+ * Not a security control. It answers "is this the file I forecast", not "who
+ * sent it", so a plain digest is right and `lib/tokens.ts` — which is about
+ * bearer secrets — is not the place for it.
+ */
+export function fileDigest(csv: string): string {
+  return createHash('sha256').update(csv, 'utf8').digest('hex')
+}
+
+export interface CsvRow {
+  /** The line in the file, counting the header as line 1. */
+  readonly number: number
+  /** Source header → cell. Cells are trimmed; a short row's missing cells are `''`. */
+  readonly values: Readonly<Record<string, string>>
+}
+
+export interface ParsedCsv {
+  readonly headers: readonly string[]
+  readonly rows: readonly CsvRow[]
+}
+
+export class CsvFormatError extends Error {
+  constructor(message: string) {
+    super(message)
+    this.name = 'CsvFormatError'
+  }
+}
+
+/** A leading byte-order mark, which a spreadsheet export routinely writes. */
+const BYTE_ORDER_MARK = /^/u
+
+/**
+ * Splits the text into physical cells, honouring quoted fields.
+ *
+ * A quote inside a quoted field is doubled (`""`), and a newline inside one does
+ * not end the record — which is why this cannot be a `split` on lines.
+ */
+function splitRows(text: string): string[][] {
+  const rows: string[][] = []
+  let row: string[] = []
+  let cell = ''
+  let inQuotes = false
+  const input = text.replace(BYTE_ORDER_MARK, '')
+
+  for (let index = 0; index < input.length; index += 1) {
+    const character = input[index]
+
+    if (inQuotes) {
+      if (character !== '"') {
+        cell += character
+        continue
+      }
+
+      if (input[index + 1] === '"') {
+        cell += '"'
+        index += 1
+      } else {
+        inQuotes = false
+      }
+
+      continue
+    }
+
+    if (character === '"') {
+      inQuotes = true
+      continue
+    }
+
+    if (character === ',') {
+      row.push(cell)
+      cell = ''
+      continue
+    }
+
+    if (character === '\n') {
+      row.push(cell)
+      rows.push(row)
+      row = []
+      cell = ''
+      continue
+    }
+
+    // A bare CR belongs to the CRLF that ends a line; a lone one is not a cell.
+    if (character === '\r') {
+      continue
+    }
+
+    cell += character
+  }
+
+  if (cell.length > 0 || row.length > 0) {
+    row.push(cell)
+    rows.push(row)
+  }
+
+  return rows
+}
+
+function isBlank(cells: readonly string[]): boolean {
+  return cells.every((cell) => cell.trim().length === 0)
+}
+
+/**
+ * @throws CsvFormatError when the file has no header row, when a header is
+ *   blank, or when two headers are the same. Each of those makes a `column_map`
+ *   unanswerable rather than merely inconvenient.
+ */
+export function parseCsv(text: string): ParsedCsv {
+  const [headerCells, ...dataCells] = splitRows(text)
+
+  if (headerCells === undefined || isBlank(headerCells)) {
+    throw new CsvFormatError('The file has no header row')
+  }
+
+  const headers = headerCells.map((header) => header.trim())
+  const blank = headers.findIndex((header) => header.length === 0)
+
+  if (blank >= 0) {
+    throw new CsvFormatError(`Column ${String(blank + 1)} of the header row has no name`)
+  }
+
+  const duplicate = headers.find((header, index) => headers.indexOf(header) !== index)
+
+  if (duplicate !== undefined) {
+    throw new CsvFormatError(`The header "${duplicate}" appears more than once`)
+  }
+
+  const rows: CsvRow[] = []
+
+  dataCells.forEach((cells, index) => {
+    if (isBlank(cells)) {
+      return
+    }
+
+    const values: Record<string, string> = {}
+
+    headers.forEach((header, column) => {
+      values[header] = (cells[column] ?? '').trim()
+    })
+
+    // +2, not +1: the header is line 1 and this index counts from the line after.
+    rows.push({ number: index + 2, values })
+  })
+
+  return { headers, rows }
+}
+
+function escapeCell(value: string): string {
+  return /["\n\r,]/u.test(value) ? `"${value.replace(/"/gu, '""')}"` : value
+}
+
+/** One CSV line, terminated. Rows are written a line at a time so an export can stream. */
+export function csvLine(cells: readonly string[]): string {
+  return `${cells.map(escapeCell).join(',')}\n`
+}
+
+export function serialiseCsv(headers: readonly string[], rows: readonly (readonly string[])[]): string {
+  return [headers, ...rows].map(csvLine).join('')
+}

package/src/modules/import-export/drafts.ts

@@ -0,0 +1,182 @@
+import {
+  ACCOUNT_TYPES,
+  COMPANY_STAGES,
+  ICP_FITS,
+  INFLUENCE_LEVELS,
+  PREFERRED_CHANNELS,
+  RELATIONSHIP_LEVELS,
+  SIZE_BANDS,
+} from '@kelpie/schemas'
+
+import { normaliseDomain, normaliseEmail } from '../../lib/normalisation.ts'
+import { moneyToCents, splitList } from './mapping.ts'
+import { canonicalEnum } from './validation.ts'
+
+/**
+ * A mapped row, as the columns a write would set.
+ *
+ * **A mapped column whose cell is blank is left out.** `import-export.md` says
+ * update overwrites mapped fields, and the tempting reading is that a blank cell
+ * blanks the field. It does not: a partial export with an empty Summary column
+ * would then erase every summary in the workspace, and no author of a
+ * spreadsheet expects an empty cell to delete anything. A blank cell says
+ * nothing about the field. Clearing one stays a job for the record's own PATCH.
+ *
+ * Pure. Every value here comes from the row; nothing is looked up.
+ */
+
+/** A field absent from a draft is not written. Present fields are written as given. */
+export interface CompanyDraft {
+  readonly name?: string
+  readonly domain?: string
+  readonly industry?: string
+  readonly description?: string
+  readonly stage?: string
+  readonly sizeBand?: string
+  readonly hq?: string
+  readonly website?: string
+  readonly accountType?: string
+  readonly icpFit?: string
+  readonly summary?: string
+  readonly tags?: readonly string[]
+}
+
+export interface PersonDraft {
+  readonly name?: string
+  readonly email?: string
+  readonly phones?: readonly string[]
+  readonly timezone?: string
+  readonly location?: string
+  readonly preferredChannel?: string
+  readonly influence?: string
+  readonly relationship?: string
+  readonly summary?: string
+  readonly tags?: readonly string[]
+}
+
+/** The deal columns a row carries directly. Company, stage, owner and people are resolved. */
+export interface DealFieldsDraft {
+  readonly name?: string
+  readonly valueCents?: number | null
+  readonly expectedClose?: string
+  readonly competitors?: readonly string[]
+  readonly risks?: string
+  readonly whyWin?: string
+  readonly summary?: string
+  readonly tags?: readonly string[]
+  readonly externalId?: string
+}
+
+/**
+ * Defaults for a Company an import invents, for columns the table requires and
+ * a CSV need not carry.
+ *
+ * `other` and `unknown` rather than the `prospect` a form submit uses: somebody
+ * filling in a web form is by definition an inbound prospect, and a row in a
+ * migration file says nothing at all about the relationship. Guessing `prospect`
+ * for ten thousand companies would put a claim in the data that nobody made.
+ * `size_band` has no unknown, so it takes the smallest.
+ */
+export const NEW_COMPANY_DEFAULTS = {
+  stage: 'other',
+  sizeBand: '1-10',
+  accountType: 'other',
+  icpFit: 'unknown',
+} as const
+
+/** Defaults for a Person an import invents. Matches what a form submit uses. */
+export const NEW_PERSON_DEFAULTS = {
+  preferredChannel: 'email',
+  influence: 'influencer',
+  relationship: 'cold',
+} as const
+
+/** The currency a deal takes when a CSV carries a value and no currency column exists. */
+export const IMPORT_DEAL_CURRENCY = 'USD'
+
+/** @returns The trimmed cell, or undefined when the column is unmapped or blank. */
+function text(mapped: Readonly<Record<string, string>>, column: string): string | undefined {
+  const value = (mapped[column] ?? '').trim()
+
+  return value.length === 0 ? undefined : value
+}
+
+function list(
+  mapped: Readonly<Record<string, string>>,
+  column: string,
+): readonly string[] | undefined {
+  if (text(mapped, column) === undefined) {
+    return undefined
+  }
+
+  return splitList(mapped[column])
+}
+
+function enumeration(
+  mapped: Readonly<Record<string, string>>,
+  column: string,
+  values: readonly string[],
+): string | undefined {
+  return canonicalEnum(values, mapped[column]) ?? undefined
+}
+
+/** Drops the keys whose value is undefined, so a draft holds only what it sets. */
+function present<T extends object>(draft: T): T {
+  return Object.fromEntries(Object.entries(draft).filter(([, value]) => value !== undefined)) as T
+}
+
+export function companyDraft(mapped: Readonly<Record<string, string>>): CompanyDraft {
+  const domain = text(mapped, 'domain')
+
+  return present<CompanyDraft>({
+    name: text(mapped, 'name'),
+    domain: domain === undefined ? undefined : (normaliseDomain(domain) ?? undefined),
+    industry: text(mapped, 'industry'),
+    description: text(mapped, 'description'),
+    stage: enumeration(mapped, 'stage', COMPANY_STAGES),
+    sizeBand: enumeration(mapped, 'size_band', SIZE_BANDS),
+    hq: text(mapped, 'hq'),
+    website: text(mapped, 'website'),
+    accountType: enumeration(mapped, 'account_type', ACCOUNT_TYPES),
+    icpFit: enumeration(mapped, 'icp_fit', ICP_FITS),
+    summary: text(mapped, 'summary'),
+    tags: list(mapped, 'tags'),
+  })
+}
+
+export function personDraft(mapped: Readonly<Record<string, string>>): PersonDraft {
+  const email = text(mapped, 'email')
+
+  return present<PersonDraft>({
+    name: text(mapped, 'name'),
+    email: email === undefined ? undefined : (normaliseEmail(email) ?? undefined),
+    phones: list(mapped, 'phones'),
+    timezone: text(mapped, 'timezone'),
+    location: text(mapped, 'location'),
+    preferredChannel: enumeration(mapped, 'preferred_channel', PREFERRED_CHANNELS),
+    influence: enumeration(mapped, 'influence', INFLUENCE_LEVELS),
+    relationship: enumeration(mapped, 'relationship', RELATIONSHIP_LEVELS),
+    summary: text(mapped, 'summary'),
+    tags: list(mapped, 'tags'),
+  })
+}
+
+/**
+ * `value` is required, so `moneyToCents` has already been checked by
+ * `validateRow` and cannot be undefined here.
+ */
+export function dealFieldsDraft(mapped: Readonly<Record<string, string>>): DealFieldsDraft {
+  const cents = moneyToCents(mapped.value)
+
+  return present<DealFieldsDraft>({
+    name: text(mapped, 'name'),
+    valueCents: cents === undefined ? undefined : cents,
+    expectedClose: text(mapped, 'expected_close'),
+    competitors: list(mapped, 'competitors'),
+    risks: text(mapped, 'risks'),
+    whyWin: text(mapped, 'why_win'),
+    summary: text(mapped, 'summary'),
+    tags: list(mapped, 'tags'),
+    externalId: text(mapped, 'external_id'),
+  })
+}

package/src/modules/import-export/exportRows.ts

@@ -0,0 +1,85 @@
+import { OBJECT_COLUMNS } from '@kelpie/schemas'
+import type { ImportObject } from '@kelpie/schemas'
+
+import { centsToMoney } from './mapping.ts'
+import type {
+  ExportCompanyRow,
+  ExportDealRow,
+  ExportPersonRow,
+  ExportPositionRow,
+} from './repository.ts'
+
+/**
+ * Records as CSV cells, in the header order of `OBJECT_COLUMNS`.
+ *
+ * An export writes what an import reads: the stage is its **slug**, which a
+ * rename leaves alone, and the deal value is the major unit rather than cents.
+ * That is what makes a Kelpie CSV round-trip through the import wizard with the
+ * `custom` source and no mapping at all — every header matches a column name
+ * exactly.
+ *
+ * Pure.
+ */
+
+/** Pipe-separated, per `import-export.md`. */
+function joinList(values: readonly string[]): string {
+  return values.join('|')
+}
+
+export function headersFor(object: ImportObject): readonly string[] {
+  return OBJECT_COLUMNS[object].map((column) => column.key)
+}
+
+export function companyCells(row: ExportCompanyRow): readonly string[] {
+  return [
+    row.name,
+    row.domain ?? '',
+    row.industry ?? '',
+    row.stage,
+    row.sizeBand,
+    row.accountType,
+    row.icpFit,
+    row.description,
+    row.summary,
+    joinList(row.tags),
+    row.website ?? '',
+    row.hq ?? '',
+  ]
+}
+
+export function personCells(row: ExportPersonRow): readonly string[] {
+  return [
+    row.name,
+    row.email ?? '',
+    row.timezone ?? '',
+    row.location ?? '',
+    row.preferredChannel,
+    row.influence,
+    row.relationship,
+    row.summary,
+    joinList(row.tags),
+    joinList(row.phones),
+  ]
+}
+
+export function positionCells(row: ExportPositionRow): readonly string[] {
+  return [row.personEmail ?? '', row.companyDomain ?? '', row.title]
+}
+
+export function dealCells(row: ExportDealRow, personEmails: readonly string[]): readonly string[] {
+  return [
+    row.name,
+    row.companyDomain ?? '',
+    row.stageSlug,
+    centsToMoney(row.valueCents),
+    row.ownerEmail ?? '',
+    row.expectedClose ?? '',
+    joinList(personEmails),
+    joinList(row.competitors),
+    row.risks,
+    row.whyWin,
+    row.summary,
+    joinList(row.tags),
+    row.externalId ?? '',
+  ]
+}

package/src/modules/import-export/index.ts

@@ -0,0 +1,49 @@
+import type { KelpieModule } from '../../runtime/module.ts'
+import { createActivityRecorder } from '../activities/index.ts'
+import { mountImportExportRoutes } from './routes.ts'
+import * as schema from './schema.ts'
+import { createImportExportService } from './service.ts'
+import { registerImportExportTools } from './tools.ts'
+
+/**
+ * Import and export: CSV in and out, per `import-export.md`.
+ *
+ * It requires every object it writes, and `pipelines` because a deal row names a
+ * stage by slug and has to resolve it against this workspace's own board.
+ * `activities` is the timeline entry each imported record carries, written in
+ * the same transaction as the record itself.
+ *
+ * The MCP tools `import-export.md` lists are not registered: there is no MCP
+ * endpoint yet, so a tool would be a definition nothing mounts. They land with
+ * the MCP server, against these same services.
+ */
+export function createImportExportModule(migrationsDirectory: string): KelpieModule {
+  return {
+    id: 'import-export',
+    requires: ['people', 'companies', 'positions', 'pipelines', 'deals', 'activities'],
+
+    register(context) {
+      const service = createImportExportService({
+        db: context.db,
+        transaction: context.transaction,
+        createId: context.createId,
+        now: context.now,
+        recordActivity: createActivityRecorder({
+          createId: context.createId,
+          now: context.now,
+        }),
+        log: context.log,
+      })
+
+      context.schema(schema, migrationsDirectory)
+
+      context.routes((router) => {
+        mountImportExportRoutes(router, { db: context.db, now: context.now, service })
+      })
+
+      registerImportExportTools(context.mcp, service)
+
+      return Promise.resolve()
+    },
+  }
+}

package/src/modules/import-export/mapping.ts

@@ -0,0 +1,154 @@
+import { LIST_COLUMNS } from '@kelpie/schemas'
+import type { ImportColumnMap, MatchKeyOption } from '@kelpie/schemas'
+
+import { normaliseDomain, normaliseEmail } from '../../lib/normalisation.ts'
+
+/**
+ * Turning a CSV row into Kelpie values, and turning either a row or a stored
+ * record into the string a match key compares on.
+ *
+ * One key builder serves both sides. A row and the record it might match have to
+ * reduce to the same string or the match is a coin toss, and two builders in two
+ * files is how they stop agreeing. Pure.
+ */
+
+/** Cents per major unit. The `value` column is dollars, not cents — see `moneyToCents`. */
+const CENTS_PER_UNIT = 100
+
+/**
+ * Applies a column map to a source row.
+ *
+ * @returns Kelpie column → value, for the mapped columns only. An unmapped
+ *   column is absent rather than `''`, so "the file has no such column" and "the
+ *   file has one and this cell is empty" stay different facts.
+ */
+export function mapRow(
+  values: Readonly<Record<string, string>>,
+  columnMap: ImportColumnMap,
+): Readonly<Record<string, string>> {
+  const mapped: Record<string, string> = {}
+
+  for (const [column, header] of Object.entries(columnMap)) {
+    if (header === null) {
+      continue
+    }
+
+    mapped[column] = values[header] ?? ''
+  }
+
+  return mapped
+}
+
+/** A pipe-separated cell (`a|b|c`), per `import-export.md`. Blank entries are dropped. */
+export function splitList(value: string | undefined): readonly string[] {
+  return (value ?? '')
+    .split('|')
+    .map((item) => item.trim())
+    .filter((item) => item.length > 0)
+}
+
+/** Whether a Kelpie column carries a pipe-separated list. */
+export function isListColumn(column: string): boolean {
+  return LIST_COLUMNS.has(column)
+}
+
+/**
+ * The normalised form of one match-key component.
+ *
+ * Emails and domains go through the same normalisation a write does, so a row
+ * saying `HTTPS://Acme.com/` matches the company stored as `acme.com`. Names and
+ * titles are compared case-insensitively, which is how a person reads them.
+ * `external_id` is left alone: it is an opaque token from another system, and
+ * two ids differing only in case are two ids.
+ */
+function normaliseComponent(column: string, value: string): string | null {
+  if (column === 'email' || column === 'person_email' || column === 'owner_email') {
+    return normaliseEmail(value)
+  }
+
+  if (column === 'domain' || column === 'company_domain') {
+    return normaliseDomain(value)
+  }
+
+  if (column === 'external_id') {
+    const trimmed = value.trim()
+
+    return trimmed.length === 0 ? null : trimmed
+  }
+
+  const folded = value.trim().toLowerCase()
+
+  return folded.length === 0 ? null : folded
+}
+
+/**
+ * The values a match key is built from, by canonical column name.
+ *
+ * A mapped CSV row is already this shape. A stored record is reduced to it by
+ * the repository, which is what lets both sides produce the same key.
+ */
+export type MatchKeyParts = Readonly<Record<string, string | null | undefined>>
+
+/**
+ * @returns The comparable key, or null when any component is missing or blank.
+ *   Null means "this row cannot be matched", which is a row error rather than a
+ *   silent match against everything else that is also missing that column.
+ */
+export function buildMatchKey(matchKey: MatchKeyOption, parts: MatchKeyParts): string | null {
+  const components: string[] = []
+
+  for (const column of matchKey.columns) {
+    const raw = parts[column]
+
+    if (raw === null || raw === undefined) {
+      return null
+    }
+
+    const normalised = normaliseComponent(column, raw)
+
+    if (normalised === null) {
+      return null
+    }
+
+    components.push(normalised)
+  }
+
+  return `${matchKey.id}:${components.join('|')}`
+}
+
+/**
+ * Reads the `value` column as integer cents.
+ *
+ * **The column is the major unit**, not cents: it is what a person types into a
+ * spreadsheet and what HubSpot's `Amount` and Salesforce's `Amount` both carry.
+ * An export writes the same way, so a Kelpie CSV round-trips. Getting this
+ * backwards is a hundredfold error in every number, which is why it is one
+ * function with one test rather than a conversion at each call site.
+ *
+ * Currency symbols and thousands separators are stripped: a spreadsheet
+ * formatted as currency exports `$1,200.00`, and refusing that would fail rows
+ * whose meaning is not in doubt.
+ *
+ * @returns undefined when the cell is not a number. Blank is `null`, meaning no
+ *   value was given.
+ */
+export function moneyToCents(raw: string | undefined): number | null | undefined {
+  const trimmed = (raw ?? '').trim()
+
+  if (trimmed.length === 0) {
+    return null
+  }
+
+  const stripped = trimmed.replace(/[$£€\s,]/gu, '')
+
+  if (!/^-?\d+(\.\d+)?$/u.test(stripped)) {
+    return undefined
+  }
+
+  return Math.round(Number(stripped) * CENTS_PER_UNIT)
+}
+
+/** Renders stored cents back into the major unit an export writes. */
+export function centsToMoney(cents: number | null): string {
+  return cents === null ? '' : (cents / CENTS_PER_UNIT).toFixed(2)
+}