@fleetkeep/cli 0.8.0 → 0.9.0

package/README.md

@@ -26,6 +26,40 @@ opening a browser or starting an interactive login. `--no-browser` prints links
 for you to open yourself. Setup never starts a subscription, imports records or
 enables reminders on its own. It requires the matching setup API deployment.
 
+## Spreadsheet imports
+
+CLI 0.9.0 supports files containing up to 1,000 vehicle rows when the matching
+backend is deployed. The account's existing vehicle allowance still applies.
+The preview checks the whole selection's unique new registrations against the
+available capacity before the first batch is saved.
+
+```sh
+fleetkeep import preview vehicles.csv --selected-rows '[0,3,7]'
+fleetkeep import apply vehicles.csv --selected-rows '[0,3,7]' --confirm <digest-from-preview>
+```
+
+`--selected-rows` is a JSON array of original zero-based data-row indices,
+excluding the header. Repeat the same selection on preview and apply. Omit it
+to import all rows. The CLI rejects a changed file locally before contacting
+the API, then obtains fresh preview metadata and saves serial batches of five.
+Original spreadsheet line numbers remain attached to each outcome.
+
+Progress goes to stderr; `--json` keeps the result machine-readable on stdout.
+If a batch cannot be confirmed, the command exits unsuccessfully and reports
+only the earlier confirmed outcomes. It never silently skips rows or retries an
+uncertain write. A confirmed provider-throttling outcome also stops further
+batches. Only an explicit pre-write HTTP 429 rejection is retried, at most twice,
+using its bounded `Retry-After` delay.
+
+An explicit rerun of the identical file and selection uses stable batch keys;
+keep the same `--idempotency-key` if you supplied one. Replayed batches are
+reported separately from new results. Confirmed provider failures are stored too:
+review their outcomes and use a **new** `--idempotency-key` to deliberately retry
+confirmed failed rows. The same key returns the stored failures without another
+provider lookup. Keep the existing key when resolving an uncertain write. Older backends support the existing
+all-row import for files of at most 50 rows; selected or larger imports require
+the upgraded backend.
+
 ## Shared jobs and running costs
 
 Friendly commands return JSON, including missing-data and stale-tracking fields:
@@ -193,4 +227,12 @@ retry with the same key returns the original record instead of creating a duplic
 
 The private email-forwarding address requires `tolls:write`, including when reading it: anyone holding that address can submit statements. Read-only integrations do not receive it.
 
-CLI toll statement imports accept CSV files up to 256 KiB, the published API limit. Email forwarding accepts up to five CSV attachments of 512 KiB each. Preview and confirm the exact file digest before a CLI import is applied.
+CLI toll statement imports accept CSV files up to 256 KiB, the published API limit. Email forwarding accepts up to five PDF or CSV attachments of 512 KiB each. PDF charges always wait for approval. Preview and confirm the exact file digest before a CLI import is applied.
+
+### PDF invoices
+
+Forward the original PDF invoice to the account’s private forwarding address, or upload it in Toll Control. PDF charges wait for review before saving. CSV statements remain supported. Text-based GBP PDFs from recognised UK toll providers are supported; scanned, encrypted or ambiguous documents return an actionable error rather than guessed charges.
+
+`fleetkeep api call listTollPdfReviews` lists the signed-in account’s extracted rows and confirmation digests. Check the rows against the original invoice, then use `fleetkeep api call confirmTollPdfReview --data-file reviewed-pdf.json` with `batchId`, `confirmDigest`, `csv` and `confirmed: true`. The CSV is the reviewed structured data, not a file the customer needs to prepare. Keep the digest from the current review. Repeat confirmation is idempotent; a different account cannot read or confirm the invoice.
+
+These operations require a CLI release containing the PDF operation definitions. The backend candidate must be deployed before use.

package/fleetkeep.mjs

@@ -5,6 +5,7 @@ import { createHash } from 'node:crypto';
 import { basename, extname, resolve } from 'node:path';
 import { runApiCommand } from './api-command.mjs';
 import { runSetup } from './setup.mjs';
+import { applySpreadsheetBatches, parseSelectedRows } from './import-batches.mjs';
 import { authStatus, getAccessToken, login, logout } from './auth.mjs';
 
 const DEFAULT_API_URL = 'https://fleetkeep.co.uk';
@@ -38,8 +39,8 @@ Usage:
   fleetkeep jobs save <reference> --title <title> [--json]
   fleetkeep jobs close <job-id> [--json]
   fleetkeep jobs reopen <job-id> [--json]
-  fleetkeep import preview <file.csv|file.xlsx> [--json]
-  fleetkeep import apply <file.csv|file.xlsx> --confirm <digest> [--idempotency-key <key>] [--json]
+  fleetkeep import preview <file.csv|file.xlsx> [--selected-rows <json>] [--json]
+  fleetkeep import apply <file.csv|file.xlsx> --confirm <digest> [--selected-rows <json>] [--idempotency-key <key>] [--json]
   fleetkeep tolls forwarding [--json]
   fleetkeep tolls forwarding enable [--json]
   fleetkeep tolls import preview <file.csv> [--json]
@@ -88,11 +89,12 @@ OAuth access and refresh tokens are never accepted as command arguments.`;
 }
 
 class FleetkeepApiError extends Error {
-  constructor(status, payload) {
+  constructor(status, payload, retryAfter) {
     const code = typeof payload?.error === 'string' ? payload.error : `HTTP ${status}`;
     super(`Fleetkeep API rejected the request (${status}): ${code}`);
     this.status = status;
     this.payload = payload;
+    this.retryAfter = retryAfter;
   }
 }
 
@@ -255,7 +257,7 @@ async function request(path, { method = 'GET', body, headers: extraHeaders = {}
       ? null
       : await response.json().catch(() => ({ error: `HTTP ${response.status}` }));
   if (!response.ok) {
-    throw new FleetkeepApiError(response.status, payload);
+    throw new FleetkeepApiError(response.status, payload, response.headers.get('retry-after'));
   }
   return payload;
 }
@@ -406,7 +408,7 @@ async function spreadsheetPayload(fileArg) {
     : { sourceName, contentBase64: bytes.toString('base64') };
 }
 
-function printPreview(payload, jsonOutput, fileArg) {
+function printPreview(payload, jsonOutput, fileArg, selectedRows) {
   if (jsonOutput) {
     process.stdout.write(`${JSON.stringify(payload, null, 2)}\n`);
     return;
@@ -420,15 +422,22 @@ function printPreview(payload, jsonOutput, fileArg) {
       `Valid registrations: ${p.validRegistrations}`,
       `Invalid registrations: ${p.invalidRegistrations}`,
       `Duplicate registrations: ${p.duplicateRegistrations}`,
-      `Over self-serve limit: ${p.overSelfServeLimit ? 'yes' : 'no'}`,
+      ...(payload.batching ? [`Selected rows: ${payload.batching.totalRows}`] : []),
+      ...(payload.capacity
+        ? [
+            payload.capacity.canImport
+              ? 'Account capacity: available for this selection'
+              : `Account capacity: ${payload.capacity.error || 'selection exceeds allowance'}`
+          ]
+        : [`Over self-serve limit: ${p.overSelfServeLimit ? 'yes' : 'no'}`]),
       '',
       'No data was changed. To apply this exact file:',
-      `fleetkeep import apply ${JSON.stringify(fileArg)} --confirm ${payload.digest}`
+      `fleetkeep import apply ${JSON.stringify(fileArg)} --confirm ${payload.digest}${selectedRows ? ` --selected-rows '${JSON.stringify(selectedRows)}'` : ''}`
     ].join('\n') + '\n'
   );
 }
 
-function printImport(payload, jsonOutput) {
+function printImport(payload, jsonOutput, completed = true) {
   if (jsonOutput) {
     process.stdout.write(`${JSON.stringify(payload, null, 2)}\n`);
     return;
@@ -436,11 +445,19 @@ function printImport(payload, jsonOutput) {
   const summary = payload.summary;
   process.stdout.write(
     [
-      payload.replayed ? 'Import result replayed from the original request.' : 'Import completed.',
+      !completed
+        ? 'Import interrupted. Confirmed results:'
+        : payload.replayed
+          ? 'Import result replayed from the original request.'
+          : 'Import completed.',
+      ...(payload.replayedBatches && !payload.replayed
+        ? [`Previously confirmed batches: ${payload.replayedBatches}`]
+        : []),
       `Added: ${summary.added}`,
       `Existing vehicles updated with equipment: ${summary.updated}`,
       `Duplicates or unchanged: ${summary.duplicates}`,
       `Invalid: ${summary.invalid}`,
+      ...(summary.dvlaErrors !== undefined ? [`DVLA errors: ${summary.dvlaErrors}`] : []),
       `Database errors: ${summary.dbErrors}`
     ].join('\n') + '\n'
   );
@@ -452,7 +469,8 @@ function printImport(payload, jsonOutput) {
 function printTollForwarding(payload, jsonOutput) {
   const lines = [
     `Email forwarding: ${payload.available === false ? 'not available' : payload.enabled ? 'enabled' : 'not enabled'}`,
-    ...(payload.address ? [`Forward CSV statements to: ${payload.address}`] : []),
+    ...(payload.address ? [`Forward PDF invoices or CSV statements to: ${payload.address}`] : []),
+    'PDF charges wait for review: fleetkeep api call listTollPdfReviews',
     `Inbox checked every ${payload.pollIntervalMinutes.toLocaleString('en-GB')} minutes.`,
     ...(payload.enabled || payload.available === false
       ? []
@@ -803,11 +821,12 @@ async function main() {
 
   if (args[0] === 'import' && args[1] === 'preview') {
     const body = await spreadsheetPayload(args[2]);
+    const selectedRows = parseSelectedRows(requiredValue(args, '--selected-rows'));
     const payload = await request('/api/v1/imports/vehicles/preview', {
       method: 'POST',
-      body
+      body: { ...body, ...(selectedRows ? { selectedRows } : {}) }
     });
-    printPreview(payload, jsonOutput, args[2]);
+    printPreview(payload, jsonOutput, args[2], selectedRows);
     return;
   }
 
@@ -819,12 +838,25 @@ async function main() {
       );
     }
     const body = await spreadsheetPayload(args[2]);
-    const idempotencyKey = option(args, '--idempotency-key') || `cli:${confirmDigest}`;
-    const payload = await request('/api/v1/imports/vehicles/apply', {
-      method: 'POST',
-      body: { ...body, confirmDigest, idempotencyKey }
+    const idempotencyKey = requiredValue(args, '--idempotency-key') ?? `cli:${confirmDigest}`;
+    const selectedRows = parseSelectedRows(requiredValue(args, '--selected-rows'));
+    const applied = await applySpreadsheetBatches({
+      body,
+      confirmDigest,
+      idempotencyKey,
+      selectedRows,
+      request,
+      progress: (done, total) =>
+        process.stderr.write(
+          `Confirmed ${done.toLocaleString('en-GB')} of ${total.toLocaleString('en-GB')} rows.\n`
+        )
     });
-    printImport(payload, jsonOutput);
+    if (applied.error && jsonOutput)
+      process.stdout.write(
+        `${JSON.stringify({ ...applied.result, completed: false, error: applied.error, confirmedRows: applied.confirmedRows, totalRows: applied.totalRows }, null, 2)}\n`
+      );
+    else if (applied.result) printImport(applied.result, jsonOutput, !applied.error);
+    if (applied.error) fail(applied.error);
     return;
   }
 

package/import-batches.mjs

@@ -0,0 +1,267 @@
+import { createHash } from 'node:crypto';
+import { setTimeout as delay } from 'node:timers/promises';
+
+const summaryKeys = [
+  'total',
+  'added',
+  'updated',
+  'duplicates',
+  'invalid',
+  'dvlaErrors',
+  'dbErrors'
+];
+const countsByStatus = {
+  added: 'added',
+  updated: 'updated',
+  duplicate: 'duplicates',
+  invalid_reg: 'invalid',
+  invalid_data: 'invalid',
+  dvla_error: 'dvlaErrors',
+  db_error: 'dbErrors'
+};
+const hash = (value) => createHash('sha256').update(value).digest('hex');
+const same = (a, b) => Array.isArray(a) && JSON.stringify(a) === JSON.stringify(b);
+
+export function parseSelectedRows(value) {
+  if (value === null || value === undefined) return undefined;
+  let rows;
+  try {
+    rows = JSON.parse(value);
+  } catch {
+    throw new Error('--selected-rows requires a JSON array of original zero-based row indices.');
+  }
+  if (
+    !Array.isArray(rows) ||
+    !rows.length ||
+    rows.length > 1000 ||
+    rows.some((row) => !Number.isInteger(row) || row < 0 || row >= 1000) ||
+    new Set(rows).size !== rows.length
+  ) {
+    throw new Error('--selected-rows requires unique original row indices from 0 to 999.');
+  }
+  return rows.sort((a, b) => a - b);
+}
+
+export function spreadsheetDigest(body) {
+  return hash(
+    typeof body.csv === 'string'
+      ? Buffer.from(body.csv, 'utf8')
+      : Buffer.from(body.contentBase64, 'base64')
+  );
+}
+
+function retryMilliseconds(value, now) {
+  if (typeof value !== 'string' || !value.trim()) return null;
+  const seconds = /^\d+(?:\.\d+)?$/.test(value.trim())
+    ? Number(value)
+    : (Date.parse(value) - now) / 1000;
+  return Number.isFinite(seconds) && seconds >= 0 && seconds <= 300
+    ? Math.ceil(seconds * 1000)
+    : null;
+}
+
+async function quotaRequest(request, path, options, wait, now) {
+  for (let retries = 0; ; retries++) {
+    try {
+      return await request(path, options);
+    } catch (error) {
+      const milliseconds = retryMilliseconds(error.retryAfter, now());
+      if (
+        error.status !== 429 ||
+        error.payload?.error !== 'rate_limited' ||
+        retries >= 2 ||
+        milliseconds === null
+      )
+        throw error;
+      await wait(milliseconds);
+    }
+  }
+}
+
+function checkPreview(preview, digest, selectedRows) {
+  if (preview?.digest !== digest)
+    throw new Error('Server preview does not match the confirmed file. No import was started.');
+  const batch = preview.batching;
+  if (!batch) {
+    const count = preview.preview?.detectedVehicleCount;
+    if (selectedRows || !Number.isInteger(count) || count < 1 || count > 50) {
+      throw new Error(
+        'This server does not support the required batched/selected import. Update the backend before applying.'
+      );
+    }
+    return null;
+  }
+  const count = batch.totalRows;
+  const originalCount = preview.preview?.detectedVehicleCount;
+  if (
+    batch.batchSize !== 5 ||
+    !Number.isInteger(count) ||
+    count < 1 ||
+    count > 1000 ||
+    !Number.isInteger(originalCount) ||
+    originalCount < count ||
+    originalCount > 1000 ||
+    !Array.isArray(batch.sourceRows) ||
+    batch.sourceRows.length !== count ||
+    batch.sourceRows.some(
+      (row, i, rows) =>
+        !Number.isInteger(row) || row < 0 || row >= originalCount || (i > 0 && row <= rows[i - 1])
+    ) ||
+    !Array.isArray(batch.sourceLines) ||
+    batch.sourceLines.length !== count ||
+    batch.sourceLines.some(
+      (line, i, lines) => !Number.isInteger(line) || line < 1 || (i > 0 && line <= lines[i - 1])
+    ) ||
+    !/^[a-f0-9]{64}$/.test(preview.selectionDigest ?? '') ||
+    !same(batch.sourceRows, selectedRows ?? Array.from({ length: originalCount }, (_, i) => i))
+  ) {
+    throw new Error(
+      'Import preview has an invalid or incomplete selected-row envelope. No import was started.'
+    );
+  }
+  return batch;
+}
+
+function checkBatch(result, preview, offset) {
+  const expected = preview.batching;
+  const rows = expected.sourceRows.slice(offset, offset + 5);
+  const lines = expected.sourceLines.slice(offset, offset + 5);
+  const next = offset + rows.length < expected.totalRows ? offset + rows.length : null;
+  const batch = result?.batch;
+  if (
+    !batch ||
+    batch.offset !== offset ||
+    batch.batchSize !== 5 ||
+    batch.totalRows !== expected.totalRows ||
+    batch.selectionDigest !== preview.selectionDigest ||
+    batch.nextOffset !== next ||
+    !same(batch.sourceRows, rows) ||
+    !same(batch.sourceLines, lines) ||
+    !Array.isArray(result.outcomes) ||
+    result.outcomes.length !== rows.length ||
+    result.outcomes.some((row) => !row || typeof row !== 'object' || !Number.isInteger(row.line)) ||
+    !same(
+      result.outcomes.map((row) => row.line).sort((a, b) => a - b),
+      lines
+    ) ||
+    (result.replayed !== undefined && typeof result.replayed !== 'boolean')
+  ) {
+    throw new Error(
+      'Import batch response is incomplete or belongs to another row selection. Its result is unconfirmed; stopped without advancing.'
+    );
+  }
+  const counts = Object.fromEntries(summaryKeys.map((key) => [key, 0]));
+  counts.total = rows.length;
+  for (const row of result.outcomes) {
+    const key = Object.hasOwn(countsByStatus, row.status) ? countsByStatus[row.status] : null;
+    if (!key)
+      throw new Error(
+        'Import batch contains an unknown outcome. Its result is unconfirmed; stopped without advancing.'
+      );
+    counts[key]++;
+  }
+  if (summaryKeys.some((key) => result.summary?.[key] !== counts[key])) {
+    throw new Error(
+      'Import batch summary does not account for its rows. Its result is unconfirmed; stopped without advancing.'
+    );
+  }
+}
+
+export async function applySpreadsheetBatches({
+  body,
+  confirmDigest,
+  idempotencyKey,
+  selectedRows,
+  request,
+  progress = () => {},
+  wait = delay,
+  now = Date.now
+}) {
+  if (!/^[a-f0-9]{64}$/.test(confirmDigest ?? '') || spreadsheetDigest(body) !== confirmDigest) {
+    throw new Error('The file does not match --confirm. Preview this file again before applying.');
+  }
+  const source = { ...body, ...(selectedRows ? { selectedRows } : {}) };
+  let result = null;
+  let confirmedRows = 0;
+  let totalRows = null;
+  try {
+    const preview = await quotaRequest(
+      request,
+      '/api/v1/imports/vehicles/preview',
+      { method: 'POST', body: source },
+      wait,
+      now
+    );
+    const batch = checkPreview(preview, confirmDigest, selectedRows);
+    if ((batch || preview.capacity) && preview.capacity?.canImport !== true) {
+      throw new Error(
+        preview.capacity?.error ||
+          'This selected import exceeds the account vehicle allowance. Review the account before applying.'
+      );
+    }
+    if (!batch) {
+      result = await quotaRequest(
+        request,
+        '/api/v1/imports/vehicles/apply',
+        {
+          method: 'POST',
+          body: { ...source, confirmDigest, idempotencyKey }
+        },
+        wait,
+        now
+      );
+      return { result, error: null };
+    }
+    totalRows = batch.totalRows;
+    for (let offset = 0; offset < totalRows; offset += 5) {
+      const key = `cli:batch:${hash(JSON.stringify([idempotencyKey, confirmDigest, preview.selectionDigest, offset]))}`;
+      const applied = await quotaRequest(
+        request,
+        '/api/v1/imports/vehicles/apply',
+        {
+          method: 'POST',
+          body: {
+            ...source,
+            confirmDigest,
+            confirmSelectionDigest: preview.selectionDigest,
+            offset,
+            idempotencyKey: key
+          }
+        },
+        wait,
+        now
+      );
+      checkBatch(applied, preview, offset);
+      if (!result) {
+        const { batch: _batch, ...rest } = applied;
+        result = {
+          ...rest,
+          summary: { ...applied.summary },
+          outcomes: [...applied.outcomes],
+          replayed: Boolean(applied.replayed),
+          replayedBatches: applied.replayed ? 1 : 0
+        };
+      } else {
+        for (const key of summaryKeys) result.summary[key] += applied.summary[key];
+        result.outcomes.push(...applied.outcomes);
+        result.replayed &&= Boolean(applied.replayed);
+        result.replayedBatches += applied.replayed ? 1 : 0;
+      }
+      confirmedRows += applied.outcomes.length;
+      progress(confirmedRows, totalRows);
+      if (applied.outcomes.some((row) => row.code === 'rate_limited')) {
+        throw new Error(
+          'DVLA is limiting requests. Review the confirmed outcomes. To retry confirmed failed rows later, use a NEW --idempotency-key; the same key replays the stored result.'
+        );
+      }
+    }
+    return { result, error: null, confirmedRows, totalRows };
+  } catch (error) {
+    return {
+      result,
+      error: `${error instanceof Error ? error.message : 'Import interrupted.'} Confirmed earlier results are kept. No uncertain write was retried.`,
+      confirmedRows,
+      totalRows
+    };
+  }
+}

package/operations.mjs

@@ -300,6 +300,8 @@ const definitions = [
     'tolls:write',
     { idempotency: true }
   ],
+  ['listTollPdfReviews', 'GET', '/api/v1/tolls/pdf-reviews', 'tolls:read'],
+  ['confirmTollPdfReview', 'POST', '/api/v1/tolls/pdf-reviews', 'tolls:write'],
   ['getTollForwarding', 'GET', '/api/v1/tolls/forwarding', 'tolls:write'],
   ['enableTollForwarding', 'POST', '/api/v1/tolls/forwarding', 'tolls:write'],
   ['previewTollImport', 'POST', '/api/v1/tolls/imports/preview', 'tolls:read'],

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fleetkeep/cli",
-  "version": "0.8.0",
+  "version": "0.9.0",
   "description": "Operate every published Fleetkeep API workflow with guarded writes and machine-readable output",
   "type": "module",
   "bin": {
@@ -10,6 +10,7 @@
     "api-command.mjs",
     "auth.mjs",
     "fleetkeep.mjs",
+    "import-batches.mjs",
     "operations.mjs",
     "setup.mjs",
     "README.md"