@kelpie/server 0.1.0

package/src/modules/import-export/plan.ts

@@ -0,0 +1,311 @@
+import type {
+  ImportConflictMode,
+  ImportCounts,
+  ImportObject,
+  MatchKeyOption,
+} from '@kelpie/schemas'
+
+import { normaliseDomain, normaliseEmail } from '../../lib/normalisation.ts'
+import { companyDraft, dealFieldsDraft, personDraft } from './drafts.ts'
+import type { CompanyDraft, DealFieldsDraft, PersonDraft } from './drafts.ts'
+import { buildMatchKey, splitList } from './mapping.ts'
+import { aliasedStageSlug } from './presets.ts'
+import type { StoredRowError } from './schema.ts'
+import { validateRow } from './validation.ts'
+
+/**
+ * What a row would do, and to what.
+ *
+ * The dry run and the commit both go through here. The difference is where the
+ * lookups come from: a dry run passes one snapshot taken over the whole file, a
+ * commit passes what it just read inside its transaction. One planner means the
+ * commit cannot decide something the preview never showed.
+ *
+ * Pure. Every lookup arrives as a map.
+ */
+
+export interface ImportLookups {
+  /** Match key → the id of the record already holding it. */
+  readonly existing: ReadonlyMap<string, string>
+  readonly personIdByEmail: ReadonlyMap<string, string>
+  readonly companyIdByDomain: ReadonlyMap<string, string>
+  /** Workspace member id by the address of the user behind it. */
+  readonly memberIdByEmail: ReadonlyMap<string, string>
+  /** Deal stage id, keyed by both its slug and its folded label. */
+  readonly dealStageIdByName: ReadonlyMap<string, string>
+}
+
+export interface PlanContext {
+  readonly object: ImportObject
+  readonly matchKey: MatchKeyOption
+  readonly conflictMode: ImportConflictMode
+  readonly lookups: ImportLookups
+}
+
+/** The values a write applies, once every reference in the row has been resolved. */
+export type ImportWrite =
+  | { readonly object: 'companies'; readonly draft: CompanyDraft }
+  | { readonly object: 'people'; readonly draft: PersonDraft }
+  | {
+      readonly object: 'positions'
+      readonly personId: string
+      readonly companyId: string
+      readonly title: string
+    }
+  | {
+      readonly object: 'deals'
+      readonly draft: DealFieldsDraft
+      readonly companyId: string
+      readonly stageId: string
+      readonly ownerId: string | null
+      readonly personIds: readonly string[]
+      /**
+       * Whether the row said anything about the deal's contacts.
+       *
+       * An empty `personIds` means two different things — the column was not
+       * mapped, or it was mapped and left blank — and only a filled-in cell is
+       * the file stating the list. Without this an update would clear the
+       * contacts of every deal in a file that never carried the column.
+       */
+      readonly setsPeople: boolean
+    }
+
+export type RowPlan =
+  | { readonly action: 'error'; readonly errors: readonly StoredRowError[] }
+  /**
+   * `targetId` is null when the row matched an earlier row of the same file
+   * rather than a stored record. A dry run only counts it; by the time a commit
+   * reaches the row the earlier one has been written, so it resolves to an id.
+   */
+  | { readonly action: 'skip'; readonly key: string; readonly targetId: string | null }
+  | { readonly action: 'create'; readonly key: string; readonly write: ImportWrite }
+  | {
+      readonly action: 'update'
+      readonly key: string
+      readonly targetId: string | null
+      readonly write: ImportWrite
+    }
+
+export interface PlannedRow {
+  readonly row: number
+  readonly plan: RowPlan
+}
+
+/** A row as the planner takes it: the file line, and the row mapped to Kelpie columns. */
+export interface MappedRow {
+  readonly row: number
+  readonly mapped: Readonly<Record<string, string>>
+}
+
+function error(...errors: readonly StoredRowError[]): RowPlan {
+  return { action: 'error', errors }
+}
+
+/**
+ * The stage a deal row names, as a stage of this workspace.
+ *
+ * Its own slug first, then its label, then the vendor alias table. Slug before
+ * label because a rename leaves the slug alone, which is the whole reason an
+ * import addresses stages by slug (`pipeline_stages` in `schema.md`).
+ */
+function resolveStageId(lookups: ImportLookups, raw: string): string | undefined {
+  const folded = raw.trim().toLowerCase()
+  const direct = lookups.dealStageIdByName.get(folded)
+
+  if (direct !== undefined) {
+    return direct
+  }
+
+  const aliased = aliasedStageSlug(raw)
+
+  return aliased === undefined ? undefined : lookups.dealStageIdByName.get(aliased)
+}
+
+function planPosition(context: PlanContext, mapped: Readonly<Record<string, string>>): ImportWrite | RowPlan {
+  const email = normaliseEmail(mapped.person_email ?? '')
+  const domain = normaliseDomain(mapped.company_domain ?? '')
+  const personId = email === null ? undefined : context.lookups.personIdByEmail.get(email)
+  const companyId = domain === null ? undefined : context.lookups.companyIdByDomain.get(domain)
+  const problems: StoredRowError[] = []
+
+  if (personId === undefined) {
+    problems.push({ field: 'person_email', message: 'No person here has that address. Import people first' })
+  }
+
+  if (companyId === undefined) {
+    problems.push({
+      field: 'company_domain',
+      message: 'No company here has that domain. Import companies first',
+    })
+  }
+
+  if (personId === undefined || companyId === undefined) {
+    return error(...problems)
+  }
+
+  return { object: 'positions', personId, companyId, title: (mapped.title ?? '').trim() }
+}
+
+/**
+ * A deal's company, stage, owner and contacts.
+ *
+ * A missing company fails the row rather than creating a stub, per
+ * `import-export.md`. An `owner_email` that names nobody in the workspace also
+ * fails it: the mockup silently resolves that to the importer, and quietly
+ * moving another person's deals onto whoever ran the import is worse data than a
+ * refused row. Unmapping the column is the way to import without owners.
+ */
+function planDeal(context: PlanContext, mapped: Readonly<Record<string, string>>): ImportWrite | RowPlan {
+  const problems: StoredRowError[] = []
+  const domain = normaliseDomain(mapped.company_domain ?? '')
+  const companyId = domain === null ? undefined : context.lookups.companyIdByDomain.get(domain)
+
+  if (companyId === undefined) {
+    problems.push({
+      field: 'company_domain',
+      message: 'No company here has that domain. Import companies first',
+    })
+  }
+
+  const stageId = resolveStageId(context.lookups, mapped.stage ?? '')
+
+  if (stageId === undefined) {
+    problems.push({
+      field: 'stage',
+      message: `"${(mapped.stage ?? '').trim()}" is not a stage of this workspace's deal pipeline`,
+    })
+  }
+
+  const ownerEmail = normaliseEmail(mapped.owner_email ?? '')
+  const ownerId = ownerEmail === null ? null : (context.lookups.memberIdByEmail.get(ownerEmail) ?? null)
+
+  if (ownerEmail !== null && ownerId === null) {
+    problems.push({
+      field: 'owner_email',
+      message: `Nobody in this workspace is ${ownerEmail}. Leave the column unmapped to import without owners`,
+    })
+  }
+
+  const personIds: string[] = []
+
+  for (const raw of splitList(mapped.person_emails)) {
+    const address = normaliseEmail(raw)
+    const personId = address === null ? undefined : context.lookups.personIdByEmail.get(address)
+
+    if (personId === undefined) {
+      problems.push({ field: 'person_emails', message: `No person here has the address ${raw}` })
+      continue
+    }
+
+    personIds.push(personId)
+  }
+
+  if (problems.length > 0 || companyId === undefined || stageId === undefined) {
+    return error(...problems)
+  }
+
+  return {
+    object: 'deals',
+    draft: dealFieldsDraft(mapped),
+    companyId,
+    stageId,
+    ownerId,
+    personIds: [...new Set(personIds)],
+    setsPeople: (mapped.person_emails ?? '').trim().length > 0,
+  }
+}
+
+function isRowPlan(value: ImportWrite | RowPlan): value is RowPlan {
+  return 'action' in value
+}
+
+/** Resolves every reference the row names into the write it would perform. */
+function resolveWrite(context: PlanContext, mapped: Readonly<Record<string, string>>): ImportWrite | RowPlan {
+  switch (context.object) {
+    case 'companies':
+      return { object: 'companies', draft: companyDraft(mapped) }
+    case 'people':
+      return { object: 'people', draft: personDraft(mapped) }
+    case 'positions':
+      return planPosition(context, mapped)
+    case 'deals':
+      return planDeal(context, mapped)
+  }
+}
+
+/**
+ * @param mapped The row's cells by Kelpie column, from `mapRow`.
+ * @returns What this row would do. `create` when its key matches nothing;
+ *   otherwise the job's conflict mode decides between `skip` and `update`.
+ */
+export function planRow(context: PlanContext, mapped: Readonly<Record<string, string>>): RowPlan {
+  const invalid = validateRow(context.object, context.matchKey, mapped)
+
+  if (invalid.length > 0) {
+    return error(...invalid)
+  }
+
+  const write = resolveWrite(context, mapped)
+
+  if (isRowPlan(write)) {
+    return write
+  }
+
+  const key = buildMatchKey(context.matchKey, mapped)
+
+  // Unreachable while every match-key column is required, which `validateRow`
+  // enforces. Kept because that coupling lives in another file.
+  if (key === null) {
+    return error({
+      field: context.matchKey.columns[0] ?? 'match_key',
+      message: 'The key columns are not all filled in, so this row cannot be matched',
+    })
+  }
+
+  const targetId = context.lookups.existing.get(key)
+
+  if (targetId === undefined) {
+    return { action: 'create', key, write }
+  }
+
+  // An empty string is the placeholder an in-file match carries: the record it
+  // matched has not been written yet, so there is no id to report.
+  const resolved = targetId.length === 0 ? null : targetId
+
+  return context.conflictMode === 'update'
+    ? { action: 'update', key, targetId: resolved, write }
+    : { action: 'skip', key, targetId: resolved }
+}
+
+/**
+ * Plans a whole file.
+ *
+ * A row that would create a record registers its key, so a later row carrying
+ * the same key reads as a match rather than as a second create. That is the
+ * in-file duplicate the mockup handles too, and without it a file listing one
+ * company twice reports two creates and performs one.
+ */
+export function planRows(context: PlanContext, rows: readonly MappedRow[]): readonly PlannedRow[] {
+  const existing = new Map(context.lookups.existing)
+  const withOverlay: PlanContext = { ...context, lookups: { ...context.lookups, existing } }
+
+  return rows.map((row) => {
+    const plan = planRow(withOverlay, row.mapped)
+
+    if (plan.action === 'create') {
+      existing.set(plan.key, '')
+    }
+
+    return { row: row.row, plan }
+  })
+}
+
+export function countPlans(rows: readonly PlannedRow[]): ImportCounts {
+  const counts = { total: rows.length, create: 0, update: 0, skip: 0, error: 0 }
+
+  for (const { plan } of rows) {
+    counts[plan.action] += 1
+  }
+
+  return counts
+}

package/src/modules/import-export/presets.ts

@@ -0,0 +1,183 @@
+import { OBJECT_COLUMNS } from '@kelpie/schemas'
+import type { ImportColumnMap, ImportObject, ImportSource } from '@kelpie/schemas'
+
+/**
+ * Source packs: the header a vendor's CSV export uses for each Kelpie column,
+ * and the stage names their pipelines ship with.
+ *
+ * Server-side rather than in `@kelpie/schemas`, because a caller sends a
+ * `column_map` and never needs to know how one was derived. The endpoint derives
+ * it when the request omits it, which is what keeps an agent from having to
+ * reimplement any of this.
+ *
+ * Ported from `mockups/src/data/importExport.ts`. Pure.
+ */
+
+type SourcePreset = Partial<Record<ImportObject, Readonly<Record<string, string>>>>
+
+export const SOURCE_PRESETS: Readonly<Record<ImportSource, SourcePreset>> = {
+  custom: {},
+  hubspot: {
+    companies: {
+      name: 'Name',
+      domain: 'Company Domain Name',
+      industry: 'Industry',
+      website: 'Website URL',
+      hq: 'City',
+      description: 'Description',
+    },
+    people: {
+      name: 'Full Name',
+      email: 'Email',
+      phones: 'Phone Number',
+      location: 'City',
+    },
+    positions: {
+      person_email: 'Email',
+      company_domain: 'Company Domain Name',
+      title: 'Job Title',
+    },
+    deals: {
+      name: 'Deal Name',
+      company_domain: 'Associated Company Domain',
+      stage: 'Deal Stage',
+      value: 'Amount',
+      owner_email: 'Owner Email',
+      expected_close: 'Close Date',
+      external_id: 'Record ID',
+    },
+  },
+  salesforce: {
+    companies: {
+      name: 'Account Name',
+      domain: 'Website',
+      industry: 'Industry',
+      hq: 'Billing City',
+      description: 'Description',
+    },
+    people: {
+      name: 'Full Name',
+      email: 'Email',
+      phones: 'Phone',
+      location: 'Mailing City',
+    },
+    positions: {
+      person_email: 'Email',
+      company_domain: 'Account Website',
+      title: 'Title',
+    },
+    deals: {
+      name: 'Opportunity Name',
+      company_domain: 'Account Website',
+      stage: 'Stage',
+      value: 'Amount',
+      owner_email: 'Owner Email',
+      expected_close: 'Close Date',
+      external_id: 'Opportunity ID',
+    },
+  },
+}
+
+/**
+ * Vendor deal-stage names, reduced to the slug of a Kelpie starter stage.
+ *
+ * This is an alias table, not the answer: the slug it produces is looked up
+ * against the workspace's own deal pipeline, which a workspace is free to have
+ * renamed or rebuilt. A slug with no stage behind it is a row error.
+ *
+ * Keys are compared after the normalisation in `stageAliasKeys`, so one entry
+ * covers `Closed Won`, `closedwon` and `closed_won`.
+ */
+export const DEAL_STAGE_ALIASES: Readonly<Record<string, string>> = {
+  qualifying: 'qualifying',
+  qualification: 'qualifying',
+  prospecting: 'qualifying',
+  appointment_scheduled: 'qualifying',
+  qualified_to_buy: 'qualifying',
+  proposal: 'proposal',
+  presentation_scheduled: 'proposal',
+  proposal_price_quote: 'proposal',
+  value_proposition: 'proposal',
+  negotiation: 'negotiation',
+  negotiation_review: 'negotiation',
+  decision_maker_bought_in: 'negotiation',
+  perception_analysis: 'negotiation',
+  won: 'won',
+  closed_won: 'won',
+  lost: 'lost',
+  closed_lost: 'lost',
+}
+
+/**
+ * A stage name with every separator removed.
+ *
+ * Vendors write one stage several ways — `Closed Won`, `closed_won`,
+ * `CLOSEDWON` — and squashing all of them to `closedwon` makes them one key. It
+ * is the only form that works on `closedwon`, which has no separator left to
+ * split on.
+ */
+function squash(raw: string): string {
+  return raw.trim().toLowerCase().replace(/[\s_\-/]+/gu, '')
+}
+
+/**
+ * The alias table indexed by squashed name, built once.
+ *
+ * Two entries squashing to the same key would be one entry with the last
+ * definition winning, so the build refuses it rather than silently dropping an
+ * alias somebody wrote on purpose.
+ */
+const ALIASES_BY_SQUASHED_NAME: ReadonlyMap<string, string> = ((): ReadonlyMap<string, string> => {
+  const index = new Map<string, string>()
+
+  for (const [name, slug] of Object.entries(DEAL_STAGE_ALIASES)) {
+    const key = squash(name)
+    const existing = index.get(key)
+
+    if (existing !== undefined && existing !== slug) {
+      throw new Error(`Deal stage aliases "${name}" and another both reduce to "${key}"`)
+    }
+
+    index.set(key, slug)
+  }
+
+  return index
+})()
+
+/** @returns The Kelpie stage slug a vendor name aliases to, or undefined. */
+export function aliasedStageSlug(raw: string): string | undefined {
+  return ALIASES_BY_SQUASHED_NAME.get(squash(raw))
+}
+
+/**
+ * The column map to use when a request sends none.
+ *
+ * The source's preset wins, then an exact header match ignoring case, then the
+ * column is left unmapped. Preset first because a HubSpot file has both a
+ * `Name` and a `Company Domain Name`, and the preset knows which is which.
+ *
+ * @param headers The file's own header row.
+ */
+export function defaultColumnMap(
+  source: ImportSource,
+  object: ImportObject,
+  headers: readonly string[],
+): ImportColumnMap {
+  const preset = SOURCE_PRESETS[source][object] ?? {}
+  const present = new Set(headers)
+  const map: Record<string, string | null> = {}
+
+  for (const column of OBJECT_COLUMNS[object]) {
+    const fromPreset = preset[column.key]
+
+    if (fromPreset !== undefined && present.has(fromPreset)) {
+      map[column.key] = fromPreset
+      continue
+    }
+
+    map[column.key] =
+      headers.find((header) => header.toLowerCase() === column.key.toLowerCase()) ?? null
+  }
+
+  return map
+}