@fleetkeep/cli 0.8.1 → 0.9.0

package/README.md

@@ -26,6 +26,40 @@ opening a browser or starting an interactive login. `--no-browser` prints links
 for you to open yourself. Setup never starts a subscription, imports records or
 enables reminders on its own. It requires the matching setup API deployment.
 
+## Spreadsheet imports
+
+CLI 0.9.0 supports files containing up to 1,000 vehicle rows when the matching
+backend is deployed. The account's existing vehicle allowance still applies.
+The preview checks the whole selection's unique new registrations against the
+available capacity before the first batch is saved.
+
+```sh
+fleetkeep import preview vehicles.csv --selected-rows '[0,3,7]'
+fleetkeep import apply vehicles.csv --selected-rows '[0,3,7]' --confirm <digest-from-preview>
+```
+
+`--selected-rows` is a JSON array of original zero-based data-row indices,
+excluding the header. Repeat the same selection on preview and apply. Omit it
+to import all rows. The CLI rejects a changed file locally before contacting
+the API, then obtains fresh preview metadata and saves serial batches of five.
+Original spreadsheet line numbers remain attached to each outcome.
+
+Progress goes to stderr; `--json` keeps the result machine-readable on stdout.
+If a batch cannot be confirmed, the command exits unsuccessfully and reports
+only the earlier confirmed outcomes. It never silently skips rows or retries an
+uncertain write. A confirmed provider-throttling outcome also stops further
+batches. Only an explicit pre-write HTTP 429 rejection is retried, at most twice,
+using its bounded `Retry-After` delay.
+
+An explicit rerun of the identical file and selection uses stable batch keys;
+keep the same `--idempotency-key` if you supplied one. Replayed batches are
+reported separately from new results. Confirmed provider failures are stored too:
+review their outcomes and use a **new** `--idempotency-key` to deliberately retry
+confirmed failed rows. The same key returns the stored failures without another
+provider lookup. Keep the existing key when resolving an uncertain write. Older backends support the existing
+all-row import for files of at most 50 rows; selected or larger imports require
+the upgraded backend.
+
 ## Shared jobs and running costs
 
 Friendly commands return JSON, including missing-data and stale-tracking fields:

package/fleetkeep.mjs

@@ -5,6 +5,7 @@ import { createHash } from 'node:crypto';
 import { basename, extname, resolve } from 'node:path';
 import { runApiCommand } from './api-command.mjs';
 import { runSetup } from './setup.mjs';
+import { applySpreadsheetBatches, parseSelectedRows } from './import-batches.mjs';
 import { authStatus, getAccessToken, login, logout } from './auth.mjs';
 
 const DEFAULT_API_URL = 'https://fleetkeep.co.uk';
@@ -38,8 +39,8 @@ Usage:
   fleetkeep jobs save <reference> --title <title> [--json]
   fleetkeep jobs close <job-id> [--json]
   fleetkeep jobs reopen <job-id> [--json]
-  fleetkeep import preview <file.csv|file.xlsx> [--json]
-  fleetkeep import apply <file.csv|file.xlsx> --confirm <digest> [--idempotency-key <key>] [--json]
+  fleetkeep import preview <file.csv|file.xlsx> [--selected-rows <json>] [--json]
+  fleetkeep import apply <file.csv|file.xlsx> --confirm <digest> [--selected-rows <json>] [--idempotency-key <key>] [--json]
   fleetkeep tolls forwarding [--json]
   fleetkeep tolls forwarding enable [--json]
   fleetkeep tolls import preview <file.csv> [--json]
@@ -88,11 +89,12 @@ OAuth access and refresh tokens are never accepted as command arguments.`;
 }
 
 class FleetkeepApiError extends Error {
-  constructor(status, payload) {
+  constructor(status, payload, retryAfter) {
     const code = typeof payload?.error === 'string' ? payload.error : `HTTP ${status}`;
     super(`Fleetkeep API rejected the request (${status}): ${code}`);
     this.status = status;
     this.payload = payload;
+    this.retryAfter = retryAfter;
   }
 }
 
@@ -255,7 +257,7 @@ async function request(path, { method = 'GET', body, headers: extraHeaders = {}
       ? null
       : await response.json().catch(() => ({ error: `HTTP ${response.status}` }));
   if (!response.ok) {
-    throw new FleetkeepApiError(response.status, payload);
+    throw new FleetkeepApiError(response.status, payload, response.headers.get('retry-after'));
   }
   return payload;
 }
@@ -406,7 +408,7 @@ async function spreadsheetPayload(fileArg) {
     : { sourceName, contentBase64: bytes.toString('base64') };
 }
 
-function printPreview(payload, jsonOutput, fileArg) {
+function printPreview(payload, jsonOutput, fileArg, selectedRows) {
   if (jsonOutput) {
     process.stdout.write(`${JSON.stringify(payload, null, 2)}\n`);
     return;
@@ -420,15 +422,22 @@ function printPreview(payload, jsonOutput, fileArg) {
       `Valid registrations: ${p.validRegistrations}`,
       `Invalid registrations: ${p.invalidRegistrations}`,
       `Duplicate registrations: ${p.duplicateRegistrations}`,
-      `Over self-serve limit: ${p.overSelfServeLimit ? 'yes' : 'no'}`,
+      ...(payload.batching ? [`Selected rows: ${payload.batching.totalRows}`] : []),
+      ...(payload.capacity
+        ? [
+            payload.capacity.canImport
+              ? 'Account capacity: available for this selection'
+              : `Account capacity: ${payload.capacity.error || 'selection exceeds allowance'}`
+          ]
+        : [`Over self-serve limit: ${p.overSelfServeLimit ? 'yes' : 'no'}`]),
       '',
       'No data was changed. To apply this exact file:',
-      `fleetkeep import apply ${JSON.stringify(fileArg)} --confirm ${payload.digest}`
+      `fleetkeep import apply ${JSON.stringify(fileArg)} --confirm ${payload.digest}${selectedRows ? ` --selected-rows '${JSON.stringify(selectedRows)}'` : ''}`
     ].join('\n') + '\n'
   );
 }
 
-function printImport(payload, jsonOutput) {
+function printImport(payload, jsonOutput, completed = true) {
   if (jsonOutput) {
     process.stdout.write(`${JSON.stringify(payload, null, 2)}\n`);
     return;
@@ -436,11 +445,19 @@ function printImport(payload, jsonOutput) {
   const summary = payload.summary;
   process.stdout.write(
     [
-      payload.replayed ? 'Import result replayed from the original request.' : 'Import completed.',
+      !completed
+        ? 'Import interrupted. Confirmed results:'
+        : payload.replayed
+          ? 'Import result replayed from the original request.'
+          : 'Import completed.',
+      ...(payload.replayedBatches && !payload.replayed
+        ? [`Previously confirmed batches: ${payload.replayedBatches}`]
+        : []),
       `Added: ${summary.added}`,
       `Existing vehicles updated with equipment: ${summary.updated}`,
       `Duplicates or unchanged: ${summary.duplicates}`,
       `Invalid: ${summary.invalid}`,
+      ...(summary.dvlaErrors !== undefined ? [`DVLA errors: ${summary.dvlaErrors}`] : []),
       `Database errors: ${summary.dbErrors}`
     ].join('\n') + '\n'
   );
@@ -804,11 +821,12 @@ async function main() {
 
   if (args[0] === 'import' && args[1] === 'preview') {
     const body = await spreadsheetPayload(args[2]);
+    const selectedRows = parseSelectedRows(requiredValue(args, '--selected-rows'));
     const payload = await request('/api/v1/imports/vehicles/preview', {
       method: 'POST',
-      body
+      body: { ...body, ...(selectedRows ? { selectedRows } : {}) }
     });
-    printPreview(payload, jsonOutput, args[2]);
+    printPreview(payload, jsonOutput, args[2], selectedRows);
     return;
   }
 
@@ -820,12 +838,25 @@ async function main() {
       );
     }
     const body = await spreadsheetPayload(args[2]);
-    const idempotencyKey = option(args, '--idempotency-key') || `cli:${confirmDigest}`;
-    const payload = await request('/api/v1/imports/vehicles/apply', {
-      method: 'POST',
-      body: { ...body, confirmDigest, idempotencyKey }
+    const idempotencyKey = requiredValue(args, '--idempotency-key') ?? `cli:${confirmDigest}`;
+    const selectedRows = parseSelectedRows(requiredValue(args, '--selected-rows'));
+    const applied = await applySpreadsheetBatches({
+      body,
+      confirmDigest,
+      idempotencyKey,
+      selectedRows,
+      request,
+      progress: (done, total) =>
+        process.stderr.write(
+          `Confirmed ${done.toLocaleString('en-GB')} of ${total.toLocaleString('en-GB')} rows.\n`
+        )
     });
-    printImport(payload, jsonOutput);
+    if (applied.error && jsonOutput)
+      process.stdout.write(
+        `${JSON.stringify({ ...applied.result, completed: false, error: applied.error, confirmedRows: applied.confirmedRows, totalRows: applied.totalRows }, null, 2)}\n`
+      );
+    else if (applied.result) printImport(applied.result, jsonOutput, !applied.error);
+    if (applied.error) fail(applied.error);
     return;
   }
 

package/import-batches.mjs

@@ -0,0 +1,267 @@
+import { createHash } from 'node:crypto';
+import { setTimeout as delay } from 'node:timers/promises';
+
+const summaryKeys = [
+  'total',
+  'added',
+  'updated',
+  'duplicates',
+  'invalid',
+  'dvlaErrors',
+  'dbErrors'
+];
+const countsByStatus = {
+  added: 'added',
+  updated: 'updated',
+  duplicate: 'duplicates',
+  invalid_reg: 'invalid',
+  invalid_data: 'invalid',
+  dvla_error: 'dvlaErrors',
+  db_error: 'dbErrors'
+};
+const hash = (value) => createHash('sha256').update(value).digest('hex');
+const same = (a, b) => Array.isArray(a) && JSON.stringify(a) === JSON.stringify(b);
+
+export function parseSelectedRows(value) {
+  if (value === null || value === undefined) return undefined;
+  let rows;
+  try {
+    rows = JSON.parse(value);
+  } catch {
+    throw new Error('--selected-rows requires a JSON array of original zero-based row indices.');
+  }
+  if (
+    !Array.isArray(rows) ||
+    !rows.length ||
+    rows.length > 1000 ||
+    rows.some((row) => !Number.isInteger(row) || row < 0 || row >= 1000) ||
+    new Set(rows).size !== rows.length
+  ) {
+    throw new Error('--selected-rows requires unique original row indices from 0 to 999.');
+  }
+  return rows.sort((a, b) => a - b);
+}
+
+export function spreadsheetDigest(body) {
+  return hash(
+    typeof body.csv === 'string'
+      ? Buffer.from(body.csv, 'utf8')
+      : Buffer.from(body.contentBase64, 'base64')
+  );
+}
+
+function retryMilliseconds(value, now) {
+  if (typeof value !== 'string' || !value.trim()) return null;
+  const seconds = /^\d+(?:\.\d+)?$/.test(value.trim())
+    ? Number(value)
+    : (Date.parse(value) - now) / 1000;
+  return Number.isFinite(seconds) && seconds >= 0 && seconds <= 300
+    ? Math.ceil(seconds * 1000)
+    : null;
+}
+
+async function quotaRequest(request, path, options, wait, now) {
+  for (let retries = 0; ; retries++) {
+    try {
+      return await request(path, options);
+    } catch (error) {
+      const milliseconds = retryMilliseconds(error.retryAfter, now());
+      if (
+        error.status !== 429 ||
+        error.payload?.error !== 'rate_limited' ||
+        retries >= 2 ||
+        milliseconds === null
+      )
+        throw error;
+      await wait(milliseconds);
+    }
+  }
+}
+
+function checkPreview(preview, digest, selectedRows) {
+  if (preview?.digest !== digest)
+    throw new Error('Server preview does not match the confirmed file. No import was started.');
+  const batch = preview.batching;
+  if (!batch) {
+    const count = preview.preview?.detectedVehicleCount;
+    if (selectedRows || !Number.isInteger(count) || count < 1 || count > 50) {
+      throw new Error(
+        'This server does not support the required batched/selected import. Update the backend before applying.'
+      );
+    }
+    return null;
+  }
+  const count = batch.totalRows;
+  const originalCount = preview.preview?.detectedVehicleCount;
+  if (
+    batch.batchSize !== 5 ||
+    !Number.isInteger(count) ||
+    count < 1 ||
+    count > 1000 ||
+    !Number.isInteger(originalCount) ||
+    originalCount < count ||
+    originalCount > 1000 ||
+    !Array.isArray(batch.sourceRows) ||
+    batch.sourceRows.length !== count ||
+    batch.sourceRows.some(
+      (row, i, rows) =>
+        !Number.isInteger(row) || row < 0 || row >= originalCount || (i > 0 && row <= rows[i - 1])
+    ) ||
+    !Array.isArray(batch.sourceLines) ||
+    batch.sourceLines.length !== count ||
+    batch.sourceLines.some(
+      (line, i, lines) => !Number.isInteger(line) || line < 1 || (i > 0 && line <= lines[i - 1])
+    ) ||
+    !/^[a-f0-9]{64}$/.test(preview.selectionDigest ?? '') ||
+    !same(batch.sourceRows, selectedRows ?? Array.from({ length: originalCount }, (_, i) => i))
+  ) {
+    throw new Error(
+      'Import preview has an invalid or incomplete selected-row envelope. No import was started.'
+    );
+  }
+  return batch;
+}
+
+function checkBatch(result, preview, offset) {
+  const expected = preview.batching;
+  const rows = expected.sourceRows.slice(offset, offset + 5);
+  const lines = expected.sourceLines.slice(offset, offset + 5);
+  const next = offset + rows.length < expected.totalRows ? offset + rows.length : null;
+  const batch = result?.batch;
+  if (
+    !batch ||
+    batch.offset !== offset ||
+    batch.batchSize !== 5 ||
+    batch.totalRows !== expected.totalRows ||
+    batch.selectionDigest !== preview.selectionDigest ||
+    batch.nextOffset !== next ||
+    !same(batch.sourceRows, rows) ||
+    !same(batch.sourceLines, lines) ||
+    !Array.isArray(result.outcomes) ||
+    result.outcomes.length !== rows.length ||
+    result.outcomes.some((row) => !row || typeof row !== 'object' || !Number.isInteger(row.line)) ||
+    !same(
+      result.outcomes.map((row) => row.line).sort((a, b) => a - b),
+      lines
+    ) ||
+    (result.replayed !== undefined && typeof result.replayed !== 'boolean')
+  ) {
+    throw new Error(
+      'Import batch response is incomplete or belongs to another row selection. Its result is unconfirmed; stopped without advancing.'
+    );
+  }
+  const counts = Object.fromEntries(summaryKeys.map((key) => [key, 0]));
+  counts.total = rows.length;
+  for (const row of result.outcomes) {
+    const key = Object.hasOwn(countsByStatus, row.status) ? countsByStatus[row.status] : null;
+    if (!key)
+      throw new Error(
+        'Import batch contains an unknown outcome. Its result is unconfirmed; stopped without advancing.'
+      );
+    counts[key]++;
+  }
+  if (summaryKeys.some((key) => result.summary?.[key] !== counts[key])) {
+    throw new Error(
+      'Import batch summary does not account for its rows. Its result is unconfirmed; stopped without advancing.'
+    );
+  }
+}
+
+export async function applySpreadsheetBatches({
+  body,
+  confirmDigest,
+  idempotencyKey,
+  selectedRows,
+  request,
+  progress = () => {},
+  wait = delay,
+  now = Date.now
+}) {
+  if (!/^[a-f0-9]{64}$/.test(confirmDigest ?? '') || spreadsheetDigest(body) !== confirmDigest) {
+    throw new Error('The file does not match --confirm. Preview this file again before applying.');
+  }
+  const source = { ...body, ...(selectedRows ? { selectedRows } : {}) };
+  let result = null;
+  let confirmedRows = 0;
+  let totalRows = null;
+  try {
+    const preview = await quotaRequest(
+      request,
+      '/api/v1/imports/vehicles/preview',
+      { method: 'POST', body: source },
+      wait,
+      now
+    );
+    const batch = checkPreview(preview, confirmDigest, selectedRows);
+    if ((batch || preview.capacity) && preview.capacity?.canImport !== true) {
+      throw new Error(
+        preview.capacity?.error ||
+          'This selected import exceeds the account vehicle allowance. Review the account before applying.'
+      );
+    }
+    if (!batch) {
+      result = await quotaRequest(
+        request,
+        '/api/v1/imports/vehicles/apply',
+        {
+          method: 'POST',
+          body: { ...source, confirmDigest, idempotencyKey }
+        },
+        wait,
+        now
+      );
+      return { result, error: null };
+    }
+    totalRows = batch.totalRows;
+    for (let offset = 0; offset < totalRows; offset += 5) {
+      const key = `cli:batch:${hash(JSON.stringify([idempotencyKey, confirmDigest, preview.selectionDigest, offset]))}`;
+      const applied = await quotaRequest(
+        request,
+        '/api/v1/imports/vehicles/apply',
+        {
+          method: 'POST',
+          body: {
+            ...source,
+            confirmDigest,
+            confirmSelectionDigest: preview.selectionDigest,
+            offset,
+            idempotencyKey: key
+          }
+        },
+        wait,
+        now
+      );
+      checkBatch(applied, preview, offset);
+      if (!result) {
+        const { batch: _batch, ...rest } = applied;
+        result = {
+          ...rest,
+          summary: { ...applied.summary },
+          outcomes: [...applied.outcomes],
+          replayed: Boolean(applied.replayed),
+          replayedBatches: applied.replayed ? 1 : 0
+        };
+      } else {
+        for (const key of summaryKeys) result.summary[key] += applied.summary[key];
+        result.outcomes.push(...applied.outcomes);
+        result.replayed &&= Boolean(applied.replayed);
+        result.replayedBatches += applied.replayed ? 1 : 0;
+      }
+      confirmedRows += applied.outcomes.length;
+      progress(confirmedRows, totalRows);
+      if (applied.outcomes.some((row) => row.code === 'rate_limited')) {
+        throw new Error(
+          'DVLA is limiting requests. Review the confirmed outcomes. To retry confirmed failed rows later, use a NEW --idempotency-key; the same key replays the stored result.'
+        );
+      }
+    }
+    return { result, error: null, confirmedRows, totalRows };
+  } catch (error) {
+    return {
+      result,
+      error: `${error instanceof Error ? error.message : 'Import interrupted.'} Confirmed earlier results are kept. No uncertain write was retried.`,
+      confirmedRows,
+      totalRows
+    };
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fleetkeep/cli",
-  "version": "0.8.1",
+  "version": "0.9.0",
   "description": "Operate every published Fleetkeep API workflow with guarded writes and machine-readable output",
   "type": "module",
   "bin": {
@@ -10,6 +10,7 @@
     "api-command.mjs",
     "auth.mjs",
     "fleetkeep.mjs",
+    "import-batches.mjs",
     "operations.mjs",
     "setup.mjs",
     "README.md"