@abyss-project/extract 0.0.0-stage → 1.0.0

package/src/types/interface/models/extraction.dto.ts

@@ -0,0 +1,399 @@
+// ──────────────────────────────────────────────
+// Extraction DTOs.
+// Mirror server-side Sequelize models, dates serialized as ISO strings.
+// ──────────────────────────────────────────────
+
+import { ExtractionWarningCode } from '../../../types/enum/extraction-warning.enum';
+import { ExtractionFeedbackIssueType } from '../../../types/enum/extraction-feedback-issue-type.enum';
+import { DocumentType } from '../../../types/enum/document-type.enum';
+import { FileScanStatus } from '../../enum/file-scan-status.enum';
+import type {
+  IFieldValue,
+  IExecutionManifest,
+  IGlobalConfidence,
+  IExecutionTrace,
+} from './extraction-plan.dto';
+
+// Re-export so consumers can import ExtractionFeedbackIssueType from the DTO barrel.
+export { ExtractionFeedbackIssueType };
+
+export type ExtractionMode = 'auto' | 'hint' | 'template';
+export type ProcessingMode = 'sync' | 'queue';
+export type ExtractionStatus = 'pending' | 'completed' | 'failed' | 'cached';
+
+export type ExtractionFeedbackStatus = 'open' | 'triaged' | 'fixed' | 'closed';
+
+export type ExtractionUsageEventType =
+  | 'extract.sync.success'
+  | 'extract.sync.cached'
+  | 'extract.sync.failed'
+  | 'extract.queue.enqueued'
+  | 'extract.queue.success'
+  | 'extract.queue.cached'
+  | 'extract.queue.failed'
+  | 'template.run.success'
+  | 'template.run.failed';
+
+export interface IExtractionBankHint {
+  bank: string;
+  version?: string;
+}
+
+export interface IExtractionTransaction {
+  date?: string;
+  description?: string;
+  amount?: number;
+  balance?: number;
+  valueDate?: string;
+  bookingDate?: string;
+  reference?: string;
+  [k: string]: unknown;
+}
+
+export interface IExtractionMetadata {
+  iban?: string;
+  bic?: string;
+  bank?: string;
+  accountHolder?: string;
+  periodStart?: string;
+  periodEnd?: string;
+  openingBalance?: number;
+  closingBalance?: number;
+  currency?: string;
+  [k: string]: unknown;
+}
+
+/**
+ * Payslip (fiche de paie) metadata, extracted generically by FR labels — no
+ * per-employer template. Amounts are integer cents; `pasRatePct` is a percent
+ * (e.g. 3.6 for 3,6 %). `period` is a stable label (ISO month or a "du … au …"
+ * span). Every field is optional: a field is present only when confidently found.
+ */
+export interface IPayslipMetadata {
+  netCents?: number;
+  grossCents?: number;
+  netTaxableCents?: number;
+  pasRatePct?: number;
+  pasAmountCents?: number;
+  period?: string;
+  employer?: string;
+  currency?: string;
+}
+
+export interface IExtractionFile {
+  id: string;
+  userId: string;
+  applicationId: string | null;
+  pdfHash: string;
+  pdfSizeBytes: number;
+  pdfMimeType: string;
+  pdfPageCount: number | null;
+  pdfFilename: string | null;
+  applicationFileId: string | null;
+  userExternalId: string | null;
+  userMetadata: Record<string, unknown> | null;
+  detectedBank: string | null;
+  detectedVersion: string | null;
+  detectedIban: string | null;
+  detectedBic: string | null;
+  detectedCurrency: string | null;
+  /**
+   * Antivirus verdict mirrored from AbyssStorage for an ARCHIVE-OWNED file
+   * only (seeded at extractionFileService.upsert from the archive-upload
+   * response — see extraction-extract.service.ts#archivePdf — and updated by
+   * the SCAN_RESULT webhook, storage-webhook.service.ts#handleScanResult).
+   * Null for a by-reference file (AbyssExtract never requested that scan, so
+   * it never mirrors a verdict for it) and for rows created before this
+   * column existed. Storage-neutral (ADR-0038): purely informational — never
+   * redacted, never gates a read.
+   */
+  scanStatus: FileScanStatus | null;
+  createdAt: string;
+  updatedAt: string;
+}
+
+// Re-export ExtractionWarningCode so consumers can import it from the dto barrel.
+export { ExtractionWarningCode };
+
+export interface IExtractionWarning {
+  code: ExtractionWarningCode;
+  data?: {
+    /** Balance mismatch data (bank_statement). */
+    actualCents?: number;
+    expectedCents?: number;
+    currency?: string;
+    /**
+     * Adapter-specific validation error/warning type string, e.g.
+     * `ACTIF_PASSIF_MISMATCH`, `TOTAL_MISMATCH`, `MRZ_INVALID`.
+     * Present when `code === ExtractionWarningCode.ADAPTER_VALIDATION_ERROR`.
+     */
+    type?: string;
+    /** Human-readable explanation from the adapter's ValidationResult. */
+    message?: string;
+    /**
+     * The withheld field's name. Present for the per-field withdrawal codes
+     * (`FIELD_BELOW_PUBLISH_FLOOR`, `FIELD_FORMAT_ONLY_DATE` #575,
+     * `FIELD_MRZ_OUT_OF_DOMAIN` #614).
+     */
+    fieldKey?: string;
+    /**
+     * The field's calibrated confidence at the time it was withheld. Present
+     * for the per-field withdrawal codes. Never the value itself.
+     */
+    confidence?: number;
+    /**
+     * The publish floor the field missed. Present only for
+     * `FIELD_BELOW_PUBLISH_FLOOR` — neither a `FIELD_FORMAT_ONLY_DATE` nor a
+     * `FIELD_MRZ_OUT_OF_DOMAIN` withdrawal is a floor comparison, and their
+     * confidence can sit above the field's floor.
+     */
+    floor?: number;
+  };
+}
+
+export interface IExtraction {
+  id: string;
+  fileId: string;
+  mode: ExtractionMode;
+  hintBank: string | null;
+  hintVersion: string | null;
+  templateId: string | null;
+  templateVersion: number | null;
+  status: ExtractionStatus;
+  strategyUsed: string | null;
+  confidence: number | null;
+  recordsCount: number | null;
+  balanceValidated: boolean | null;
+  durationMs: number | null;
+  error: string | null;
+  recordsJson: IExtractionTransaction[] | null;
+  metadataJson: IExtractionMetadata | null;
+  warningsJson: IExtractionWarning[] | null;
+  tags: string[] | null;
+  note: string | null;
+  shareToken: string | null;
+  shareExpiresAt: string | null;
+  documentType: string | null;
+  documentTypeConfidence: number | null;
+  /**
+   * Persisted per-field results. Widened (V2-P1, ADDITIVE) to the IFieldValue
+   * superset: value + confidence stay at the SAME path (old readers unaffected),
+   * and the plan path additionally surfaces chosen / sources / signals /
+   * agreement provenance. Legacy template/hints rows persist only value +
+   * confidence (still a valid IFieldValue subset on the wire).
+   */
+  fieldsJson: Record<string, IFieldValue> | null;
+  improvementConsent: boolean;
+  usableForTraining: boolean;
+  supportTicketId: string | null;
+  /** AbyssExtract-owned training copy (extraction-training bucket). Purgeable. */
+  trainingFileId: string | null;
+  /** Whether the OCR layer was used for this extraction run. */
+  usedOcr: boolean;
+  /** Number of pages processed by the OCR layer. */
+  ocrPageCount: number;
+  /**
+   * Frozen execution manifest (resolved plan + per-pass impl/version) — the
+   * reproducible recipe. Present on plan-path rows; absent (null/undefined) on
+   * legacy template/hints rows. ADDITIVE, optional.
+   */
+  executionManifest?: IExecutionManifest | null;
+  /**
+   * Auditable global confidence (stats + aggregation + per-pass contributions).
+   * Surfaced where available; optional/absent otherwise. ADDITIVE, optional.
+   */
+  globalConfidence?: IGlobalConfidence | null;
+  /**
+   * Per-pass execution trace skeleton (timing/status/impl + lazy IO refs) — P1.
+   * Present on plan-path rows; absent on legacy paths. ADDITIVE, optional.
+   */
+  executionTrace?: IExecutionTrace | null;
+  createdAt: string;
+}
+
+export interface IExtractionTemplate {
+  id: string;
+  /** Personal scope owner — null when the template is org/project-scoped. */
+  userId: string | null;
+  /** Owning organization — null for personal templates. */
+  organizationId: string | null;
+  /** Owning project — null for personal and org-wide templates. */
+  projectId: string | null;
+  /** Author inside an org/project scope — null for personal templates. */
+  createdByUserId: string | null;
+  name: string;
+  description: string | null;
+  /**
+   * Kind of document the template targets (DocumentType). Null when the
+   * template declares no type. Authoritative — spatial templates also embed
+   * `templateJson.documentType`, but this column is the source of truth.
+   */
+  documentType: DocumentType | null;
+  isActive: boolean;
+  /**
+   * Stamped (fire-and-forget) whenever an extraction run resolves the
+   * template. Null until first use.
+   */
+  lastUsedAt: string | null;
+  createdAt: string;
+  updatedAt: string;
+}
+
+/**
+ * Spatial zone on a PDF page (PDF coords: origin bottom-left, points).
+ * Restricts regex matching to a specific rectangle.
+ */
+export interface IExtractionTemplateZone {
+  name: string;
+  page: number; // 1-indexed
+  x: number;
+  y: number;
+  w: number;
+  h: number;
+}
+
+/**
+ * Typed shape of the template DSL JSON stored in `templateJson`. Mirrors
+ * `TemplateDSL` server-side. UI editors should narrow `templateJson` to this
+ * shape via `templateJson as IExtractionTemplateDsl`.
+ */
+export interface IExtractionTemplateDsl {
+  regex_iban?: string;
+  regex_bic?: string;
+  period?: { regex: string };
+  balance?: { opening_regex?: string; closing_regex?: string };
+  transactions?: {
+    row_pattern: string;
+    date_format?: string;
+    amount_locale?: 'fr' | 'en';
+    sign_rule?: 'explicit' | 'keyword';
+  };
+  skip_patterns?: string[];
+  zones?: IExtractionTemplateZone[];
+}
+
+export interface IExtractionTemplateVersion {
+  id: string;
+  templateId: string;
+  version: number;
+  templateJson: Record<string, unknown>;
+  comment: string | null;
+  /**
+   * Witness (sample) PDF reference in the dedicated /rcb-witness bucket.
+   * Null when the version carries no witness. Retrieve a download URL via
+   * `sdk.extract.templates.getWitnessUrl`.
+   */
+  witnessFileId: string | null;
+  /** Witness byte size (billed to the owner's storage pool). */
+  witnessSizeBytes: number | null;
+  /** Original witness filename. */
+  witnessFilename: string | null;
+  createdAt: string;
+}
+
+export interface IExtractionFeedback {
+  id: string;
+  extractionId: string;
+  userId: string;
+  issueType: ExtractionFeedbackIssueType;
+  fieldKey: string | null;
+  expectedValue: string | null;
+  description: string | null;
+  supportTicketId: string | null;
+  status: ExtractionFeedbackStatus;
+  createdAt: string;
+}
+
+export interface IExtractionWebhookDelivery {
+  id: string;
+  userId: string | null;
+  organizationId: string | null;
+  projectId: string | null;
+  createdByUserId: string | null;
+  webhookId: string | null;
+  jobId: string | null;
+  event: string;
+  url: string;
+  payloadJson: Record<string, unknown>;
+  attempts: number;
+  lastStatus: number | null;
+  lastError: string | null;
+  deliveredAt: string | null;
+  createdAt: string;
+  updatedAt: string;
+}
+
+// Webhook event identifier. Values mirror the AbyssApplicationEvent extraction
+// entries so a logger.log(applicationEvent: ...) and a webhook delivery
+// can never drift.
+export enum ExtractionWebhookEvent {
+  EXTRACT_COMPLETED = 'rcb.extract.completed',
+  EXTRACT_FAILED = 'rcb.extract.failed',
+  EXTRACT_LOW_CONFIDENCE = 'rcb.extract.low-confidence',
+  EXTRACT_BALANCE_KO = 'rcb.extract.balance-ko',
+  FILE_ARCHIVED = 'rcb.file.archived',
+  SHARE_MINTED = 'rcb.share.minted',
+  SHARE_REVOKED = 'rcb.share.revoked',
+  FEEDBACK_SUBMITTED = 'rcb.feedback.submitted',
+  JOB_FAILED = 'rcb.job.failed',
+  EXPORT_COMPLETED = 'rcb.export.completed',
+  EXPORT_FAILED = 'rcb.export.failed',
+}
+
+export const EXTRACTION_WEBHOOK_EVENTS: ExtractionWebhookEvent[] =
+  Object.values(ExtractionWebhookEvent);
+
+export interface IExtractionWebhook {
+  id: string;
+  userId: string | null;
+  organizationId: string | null;
+  projectId: string | null;
+  createdByUserId: string | null;
+  webhookUrl: string;
+  webhookSecret: string;
+  isEnabled: boolean;
+  name: string;
+  description: string;
+  scopes: ExtractionWebhookEvent[];
+  consecutiveFailures: number;
+  createdAt: string;
+  updatedAt: string;
+}
+
+export interface IExtractionBankVersion {
+  id: string;
+  bank: string;
+  version: string;
+  validFrom: string | null;
+  validTo: string | null;
+  detectorJson: Record<string, unknown>;
+  parserKind: string;
+  configJson: Record<string, unknown>;
+  isActive: boolean;
+  createdAt: string;
+  updatedAt: string;
+}
+
+// ─── EXTRACTION SETTINGS (P4) ────────────────────────────────────────────────
+
+/**
+ * Scope discriminator for extraction settings.
+ * 'user' = personal scope; 'org' = organization scope.
+ */
+export enum ExtractionSettingsScopeKind {
+  USER = 'user',
+  ORG = 'org',
+}
+
+/** Per-scope improvement-consent settings row. */
+export interface IExtractionSettings {
+  id: string;
+  /** Set for personal scope. XOR with organizationId. */
+  userId: string | null;
+  /** Set for organization scope. XOR with userId. */
+  organizationId: string | null;
+  /** Factory default for per-extraction improvementConsent. OFF (false) by default. */
+  improvementDefault: boolean;
+  createdAt: string;
+  updatedAt: string;
+}

package/src/types/interface/models/index.ts

@@ -0,0 +1,3 @@
+export * from './extraction.dto';
+export * from './extraction-plan.dto';
+export * from './extraction-spatial-template.dto';

package/src/types/tests/extract-bank-catalogue.spec.ts

@@ -0,0 +1,74 @@
+import {
+  EXTRACT_BANK_CATALOGUE,
+  ExtractBankVersionId,
+  getExtractBankCatalogueEntry,
+  groupExtractCatalogueByBank,
+} from '../enum/extract-bank.enum';
+
+// Snapshot of the catalogue as shipped before the rename (id, bank, version, country, type).
+const EXPECTED_ENTRIES: ReadonlyArray<readonly [string, string, string, string, string]> = [
+  ['bnp-connexis-default', 'BNP Connexis', 'default', 'FR', 'business'],
+  ['bnp-paribas-default', 'BNP Paribas', 'default', 'FR', 'retail'],
+  ['societe-generale-default', 'Société Générale', 'default', 'FR', 'retail'],
+  ['credit-agricole-default', 'Crédit Agricole', 'default', 'FR', 'retail'],
+  ['lcl-default', 'LCL', 'default', 'FR', 'retail'],
+  ['boursorama-default', 'Boursorama', 'default', 'FR', 'retail'],
+  ['la-banque-postale-default', 'La Banque Postale', 'default', 'FR', 'retail'],
+  ['caisse-epargne-default', "Caisse d'Épargne", 'default', 'FR', 'retail'],
+  ['cic-ebics-default', 'CIC', 'ebics', 'FR', 'business'],
+  ['cic-default', 'CIC', 'default', 'FR', 'retail'],
+  ['credit-mutuel-default', 'Crédit Mutuel', 'default', 'FR', 'retail'],
+  ['bred-default', 'BRED', 'default', 'FR', 'retail'],
+  ['cdn-default', 'Crédit du Nord', 'default', 'FR', 'retail'],
+  ['cde-net-default', 'CDE Net', 'default', 'FR', 'business'],
+  ['delubac-default', 'Delubac', 'default', 'FR', 'business'],
+  ['palatine-default', 'Palatine', 'default', 'FR', 'business'],
+  ['natixis-default', 'Natixis', 'default', 'FR', 'corporate'],
+  ['themis-default', 'Themis Banque', 'default', 'FR', 'business'],
+  ['rothschild-default', 'Rothschild Martin Maurel', 'default', 'FR', 'corporate'],
+  ['la-nef-v1', 'La Nef', 'v1', 'FR', 'retail'],
+  ['la-nef-v2', 'La Nef', 'v2', 'FR', 'retail'],
+  ['qonto-default', 'Qonto', 'default', 'FR', 'neobank'],
+  ['shine-default', 'Shine', 'default', 'FR', 'neobank'],
+  ['finom-default', 'Finom', 'default', 'EU', 'neobank'],
+  ['tiime-default', 'Tiime', 'default', 'FR', 'neobank'],
+  ['spendesk-default', 'Spendesk', 'default', 'FR', 'fintech'],
+  ['swan-default', 'Swan', 'default', 'FR', 'fintech'],
+  ['n26-default', 'N26', 'default', 'EU', 'neobank'],
+  ['revolut-default', 'Revolut', 'default', 'EU', 'neobank'],
+  ['wise-default', 'Wise', 'default', 'INTL', 'fintech'],
+  ['paypal-default', 'PayPal', 'default', 'INTL', 'fintech'],
+  ['airwallex-default', 'Airwallex', 'default', 'INTL', 'fintech'],
+  ['viva-wallet-default', 'Viva Wallet', 'default', 'EU', 'fintech'],
+  ['hsbc-default', 'HSBC', 'default', 'FR', 'retail'],
+  ['hsbc-uk-default', 'HSBC UK', 'default', 'UK', 'retail'],
+  ['ubs-default', 'UBS', 'default', 'CH', 'corporate'],
+  ['caixa-default', 'Caixa', 'default', 'PT', 'retail'],
+  ['jpmorgan-default', 'JPMorgan', 'default', 'US', 'corporate'],
+];
+
+describe('EXTRACT_BANK_CATALOGUE', () => {
+  it('holds the same entries as the catalogue it was renamed from', () => {
+    const actual = EXTRACT_BANK_CATALOGUE.map((e) => [e.id, e.bank, e.version, e.country, e.type]);
+    expect(actual).toEqual(EXPECTED_ENTRIES);
+  });
+
+  it('references only ids declared in ExtractBankVersionId', () => {
+    const declared = new Set<string>(Object.values(ExtractBankVersionId));
+    for (const entry of EXTRACT_BANK_CATALOGUE) {
+      expect(declared.has(entry.id)).toBe(true);
+    }
+  });
+
+  it('looks an entry up by id', () => {
+    expect(getExtractBankCatalogueEntry('qonto-default')?.bank).toBe('Qonto');
+    expect(getExtractBankCatalogueEntry('unknown')).toBeUndefined();
+  });
+
+  it('groups every entry under its bank', () => {
+    const grouped = groupExtractCatalogueByBank();
+    const total = Array.from(grouped.values()).reduce((sum, list) => sum + list.length, 0);
+    expect(total).toBe(EXTRACT_BANK_CATALOGUE.length);
+    expect(grouped.get('La Nef')?.map((e) => e.version)).toEqual(['v1', 'v2']);
+  });
+});

package/src/utils/enum.utils.ts

@@ -0,0 +1 @@
+export type ValueOf<T> = T[keyof T];

package/src/utils/error.utils.ts

@@ -0,0 +1,30 @@
+import { Code, ErrorType } from '../types';
+
+type ErrorParams = { code: Code; type: ErrorType; message?: string };
+
+export class ServiceException extends Error {
+  private readonly code: Code;
+
+  private readonly type: ErrorType;
+
+  getCode(): Code {
+    return this.code;
+  }
+
+  getType(): ErrorType {
+    return this.type;
+  }
+
+  constructor(params: ErrorParams) {
+    super(params.message);
+    this.code = params.code;
+    this.type = params.type;
+  }
+}
+
+export const handleError = (error: Error | ErrorParams, _params?: ErrorParams): Error => {
+  if (error instanceof Error) {
+    return error;
+  }
+  return new ServiceException(error);
+};

package/src/utils/extraction-webhook.utils.ts

@@ -0,0 +1,30 @@
+// ──────────────────────────────────────────────
+// Extraction webhook signature verification (server-side helper for receivers).
+//
+// The AbyssExtract queue signs every webhook delivery with HMAC-SHA256 of
+// the raw JSON body using the secret provided at extract time. Receivers
+// compare the x-rcb-signature header (format `sha256=<hex>`) against the
+// same computation. Constant-time compare prevents timing attacks.
+//
+// Lazy-requires `crypto` so the SDK stays browser-compatible when this
+// utility is never invoked (browsers won't receive webhooks anyway).
+// ──────────────────────────────────────────────
+
+export interface VerifyExtractionWebhookInput {
+  rawBody: string | Buffer;
+  signatureHeader: string | undefined;
+  secret: string;
+}
+
+export function verifyExtractionWebhook(input: VerifyExtractionWebhookInput): boolean {
+  if (!input.signatureHeader || !input.secret) return false;
+  // eslint-disable-next-line @typescript-eslint/no-require-imports
+  const crypto = require('crypto');
+  const provided = input.signatureHeader.replace(/^sha256=/, '');
+  const computed: string = crypto
+    .createHmac('sha256', input.secret)
+    .update(input.rawBody)
+    .digest('hex');
+  if (provided.length !== computed.length) return false;
+  return crypto.timingSafeEqual(Buffer.from(provided, 'hex'), Buffer.from(computed, 'hex'));
+}

package/src/utils/index.ts

@@ -0,0 +1,3 @@
+export * from './error.utils';
+export * from './enum.utils';
+export * from './extraction-webhook.utils';

package/tsconfig.check.json

@@ -0,0 +1,15 @@
+{
+  // Perimetre de VERIFICATION, distinct de celui du build.
+  //
+  // `tsconfig.json` exclut `src/**/*.spec.ts` parce qu'il sert a emettre `dist/`, ou une spec
+  // n'a rien a faire. Mais eslint s'appuie sur ce meme fichier via `parserOptions.project` :
+  // les cinq specs du paquet sortaient donc en « Parsing error: that TSConfig does not include
+  // this file », mesure du 2026-09-09, et n'etaient lintees par personne.
+  //
+  // Ce fichier sert uniquement a eslint. Il n'emet rien et ne change pas ce que `build` produit.
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "noEmit": true
+  },
+  "exclude": ["./dist/**/*"]
+}

package/tsconfig.json

@@ -0,0 +1,36 @@
+{
+  "compilerOptions": {
+    "incremental": false,
+    "target": "es2021",
+    "lib": [
+      "dom",
+      "es2021"
+    ],
+    "experimentalDecorators": true,
+    "emitDecoratorMetadata": true,
+    "module": "commonjs",
+    "rootDir": "src",
+    "moduleResolution": "node",
+    "resolveJsonModule": true,
+    "allowJs": true,
+    "declaration": true,
+    "sourceMap": true,
+    "outDir": "./dist",
+    "removeComments": true,
+    "allowSyntheticDefaultImports": true,
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "strict": true,
+    "strictNullChecks": true,
+    "strictFunctionTypes": true,
+    "strictPropertyInitialization": false,
+    "skipLibCheck": true
+  },
+  "include": [
+    "./src"
+  ],
+  "exclude": [
+    "./dist/**/*",
+    "./src/**/*.spec.ts"
+  ]
+}