@accounter/scraper-app 0.0.1

package/src/server/payload-schemas/cal.schema.ts

@@ -0,0 +1,27 @@
+import { z } from 'zod';
+
+// Matches CalTransaction from @accounter/modern-poalim-scraper
+const CalTransactionSchema = z
+  .object({
+    trnIntId: z.string(),
+    merchantName: z.string(),
+    trnPurchaseDate: z.string(),
+    trnAmt: z.number(),
+    trnCurrencySymbol: z.string(),
+    trnType: z.string(),
+    debCrdDate: z.string(),
+    amtBeforeConvAndIndex: z.number(),
+    debCrdCurrencySymbol: z.string(),
+  })
+  .loose();
+
+// One entry per (card, month) pair scraped
+const CalMonthResultSchema = z.object({
+  card: z.string(),
+  month: z.string(),
+  transactions: z.array(CalTransactionSchema),
+});
+
+export const CalPayloadSchema = z.array(CalMonthResultSchema);
+
+export type CalPayload = z.infer<typeof CalPayloadSchema>;

package/src/server/payload-schemas/currency-rates.schema.ts

@@ -0,0 +1,11 @@
+import { z } from 'zod';
+
+const CurrencyRateEntrySchema = z.object({
+  date: z.string(),
+  currency: z.enum(['USD', 'EUR', 'GBP', 'CAD', 'JPY', 'AUD', 'SEK']),
+  rate: z.number(),
+});
+
+export const CurrencyRatesPayloadSchema = z.array(CurrencyRateEntrySchema);
+
+export type CurrencyRatesPayload = z.infer<typeof CurrencyRatesPayloadSchema>;

package/src/server/payload-schemas/discount.schema.ts

@@ -0,0 +1,26 @@
+import { z } from 'zod';
+
+// Matches DiscountTransaction from @accounter/modern-poalim-scraper
+const DiscountTransactionSchema = z
+  .object({
+    OperationDate: z.string(),
+    ValueDate: z.string(),
+    OperationCode: z.string(),
+    OperationDescription: z.string(),
+    OperationAmount: z.number(),
+    BalanceAfterOperation: z.number(),
+    OperationNumber: z.number(),
+  })
+  .loose();
+
+// One entry per (accountNumber, month) pair scraped
+const DiscountMonthResultSchema = z.object({
+  accountNumber: z.string(),
+  month: z.string(),
+  balance: z.number(),
+  transactions: z.array(DiscountTransactionSchema),
+});
+
+export const DiscountPayloadSchema = z.array(DiscountMonthResultSchema);
+
+export type DiscountPayload = z.infer<typeof DiscountPayloadSchema>;

package/src/server/payload-schemas/isracard.schema.ts

@@ -0,0 +1,58 @@
+import { z } from 'zod';
+
+const TxnIsraelSchema = z
+  .object({
+    cardIndex: z.string(),
+    supplierName: z.string().nullable(),
+    dealSum: z.string().nullable(),
+    fullPurchaseDate: z.string().nullable(),
+    purchaseDate: z.string().nullable(),
+    voucherNumber: z.string().nullable(),
+    voucherNumberRatz: z.string(),
+  })
+  .loose();
+
+const TxnAbroadSchema = z
+  .object({
+    cardIndex: z.string(),
+    fullSupplierNameOutbound: z.string(),
+    dealSumOutbound: z.string().nullable(),
+    fullPurchaseDateOutbound: z.string().nullable(),
+    paymentSumOutbound: z.string(),
+    voucherNumberRatzOutbound: z.string(),
+  })
+  .loose();
+
+const CurrentCardTransactionsSchema = z
+  .object({
+    txnIsrael: z.array(TxnIsraelSchema).nullable().optional(),
+    txnAbroad: z.array(TxnAbroadSchema).nullable().optional(),
+  })
+  .loose();
+
+const IndexSchema = z
+  .object({
+    '@AllCards': z.string(),
+    CurrentCardTransactions: z.array(CurrentCardTransactionsSchema),
+  })
+  .loose();
+
+const HeaderSchema = z
+  .object({
+    Status: z.string(),
+    Message: z.string().nullable(),
+  })
+  .loose();
+
+// CardsTransactionsListBean contains Index0, Index1, Index2, … for each card.
+// We require Index0 (always present) and capture additional Index* keys via catchall.
+const CardsTransactionsListBeanSchema = z
+  .object({ cardNumberList: z.array(z.string()), Index0: IndexSchema })
+  .catchall(z.union([IndexSchema, z.unknown()]));
+
+export const IsracardPayloadSchema = z
+  .object({
+    Header: HeaderSchema,
+    CardsTransactionsListBean: CardsTransactionsListBeanSchema,
+  })
+  .loose();

package/src/server/payload-schemas/max.schema.ts

@@ -0,0 +1,27 @@
+import { z } from 'zod';
+
+// Matches MaxTransaction from @accounter/modern-poalim-scraper
+const MaxTransactionSchema = z
+  .object({
+    cardIndex: z.number(),
+    categoryId: z.number(),
+    merchantName: z.string(),
+    originalAmount: z.number(),
+    originalCurrency: z.string(),
+    purchaseDate: z.string(),
+    uid: z.string(),
+    planName: z.string(),
+    planTypeId: z.number(),
+  })
+  .loose();
+
+// Matches TransactionsAccount (MaxScrapingResult element)
+const MaxAccountSchema = z.object({
+  accountNumber: z.string(),
+  txns: z.array(MaxTransactionSchema),
+});
+
+// MaxScrapingResult = TransactionsAccount[]
+export const MaxPayloadSchema = z.array(MaxAccountSchema);
+
+export type MaxPayload = z.infer<typeof MaxPayloadSchema>;

package/src/server/payload-schemas/poalim-foreign.schema.ts

@@ -0,0 +1,30 @@
+import { z } from 'zod';
+
+const ForeignTransactionSchema = z
+  .object({
+    activityDescription: z.string(),
+    activityTypeCode: z.number(),
+    eventAmount: z.number(),
+    currencySwiftCode: z.string(),
+    currencyRate: z.number(),
+    currentBalance: z.number(),
+    referenceNumber: z.number(),
+    transactionType: z.string(),
+  })
+  .loose();
+
+const BalancesAndLimitsItemSchema = z
+  .object({
+    currencySwiftCode: z.string(),
+    currencyCode: z.number(),
+    transactions: z.array(ForeignTransactionSchema),
+  })
+  .loose();
+
+export const PoalimForeignPayloadSchema = z
+  .object({
+    balancesAndLimitsDataList: z.array(BalancesAndLimitsItemSchema),
+  })
+  .loose();
+
+export type PoalimForeignPayload = z.infer<typeof PoalimForeignPayloadSchema>;

package/src/server/payload-schemas/poalim-ils.schema.ts

@@ -0,0 +1,31 @@
+import { z } from 'zod';
+
+const IlsTransactionSchema = z
+  .object({
+    activityDescription: z.string(),
+    activityTypeCode: z.number(),
+    eventAmount: z.number(),
+    eventDate: z.number(),
+    serialNumber: z.number(),
+    transactionType: z.enum(['REGULAR', 'TODAY', 'FUTURE']),
+    currentBalance: z.number(),
+    referenceNumber: z.number(),
+  })
+  .loose();
+
+const RetrievalTransactionDataSchema = z
+  .object({
+    accountNumber: z.number(),
+    branchNumber: z.number(),
+    bankNumber: z.number(),
+  })
+  .loose();
+
+export const PoalimIlsPayloadSchema = z
+  .object({
+    transactions: z.array(IlsTransactionSchema),
+    retrievalTransactionData: RetrievalTransactionDataSchema,
+  })
+  .loose();
+
+export type PoalimIlsPayload = z.infer<typeof PoalimIlsPayloadSchema>;

package/src/server/payload-schemas/poalim-swift.schema.ts

@@ -0,0 +1,21 @@
+import { z } from 'zod';
+
+const SwiftItemSchema = z
+  .object({
+    startDate: z.number(),
+    swiftStatusCode: z.string(),
+    amount: z.number(),
+    currencyCodeCatenatedKey: z.string(),
+    chargePartyName: z.string(),
+    referenceNumber: z.string(),
+    transferCatenatedId: z.string(),
+  })
+  .loose();
+
+export const PoalimSwiftPayloadSchema = z
+  .object({
+    swiftsList: z.array(SwiftItemSchema),
+  })
+  .loose();
+
+export type PoalimSwiftPayload = z.infer<typeof PoalimSwiftPayloadSchema>;

package/src/server/scrape-runner.ts

@@ -0,0 +1,165 @@
+import { randomUUID } from 'node:crypto';
+import type { RunRecord as SerializedRunRecord, SourceRunRecord } from '../shared/types.js';
+import type { ServerMessage } from '../shared/ws-protocol.js';
+import type { ScraperUploadResult } from './gql/index.js';
+
+export const ERR_RUN_IN_PROGRESS = 'Run already in progress';
+
+/**
+ * Thrown by buildTask when a source is blocked (unknown accounts).
+ * Signals runTask to skip task-done and task-error — task-blocked was already emitted.
+ */
+export class BlockedError extends Error {
+  readonly blockedAccounts: string[];
+  constructor(blockedAccounts: string[]) {
+    super('blocked');
+    this.name = 'BlockedError';
+    this.blockedAccounts = blockedAccounts;
+  }
+}
+
+// ── Types ─────────────────────────────────────────────────────────────────────
+
+export type ScrapeTask = {
+  sourceId: string;
+  nickname: string;
+  type: string;
+  run: () => Promise<ScraperUploadResult>;
+};
+
+export type { SourceRunRecord };
+export type RunRecord = Omit<SerializedRunRecord, 'startedAt' | 'completedAt'> & {
+  startedAt: Date;
+  completedAt: Date;
+};
+
+// ── State ─────────────────────────────────────────────────────────────────────
+
+let _running = false;
+
+export function isRunning(): boolean {
+  return _running;
+}
+
+export function _resetRunState(): void {
+  _running = false;
+}
+
+// ── Runner ────────────────────────────────────────────────────────────────────
+
+export async function startRun(
+  tasks: ScrapeTask[],
+  concurrent: boolean,
+  emit: (msg: ServerMessage) => void,
+): Promise<RunRecord> {
+  if (_running) throw new Error(ERR_RUN_IN_PROGRESS);
+  _running = true;
+
+  const id = randomUUID();
+  const startedAt = new Date();
+  let errorCount = 0;
+
+  try {
+    for (const task of tasks) {
+      emit({ type: 'task-pending', sourceId: task.sourceId });
+    }
+
+    type TaskOutcome = {
+      task: ScrapeTask;
+      result: ScraperUploadResult;
+      status: 'done' | 'error' | 'blocked';
+      error?: string;
+      blockedAccounts?: string[];
+    };
+
+    const runTask = async (task: ScrapeTask): Promise<TaskOutcome> => {
+      emit({ type: 'task-running', sourceId: task.sourceId });
+      try {
+        const result = await task.run();
+        emit({
+          type: 'task-done',
+          sourceId: task.sourceId,
+          inserted: result.inserted,
+          skipped: result.skipped,
+          insertedIds: result.insertedIds,
+          ...(result.insertedTransactions != null && {
+            insertedTransactions: result.insertedTransactions,
+          }),
+          ...(result.changedTransactions != null && {
+            changedTransactions: result.changedTransactions,
+          }),
+        });
+        return { task, result, status: 'done' };
+      } catch (e) {
+        // task-blocked was already emitted by buildTask — skip task-done and task-error
+        if (e instanceof BlockedError) {
+          return {
+            task,
+            result: {
+              inserted: 0,
+              skipped: 0,
+              insertedIds: [],
+              insertedTransactions: [],
+              changedTransactions: [],
+            },
+            status: 'blocked',
+            blockedAccounts: e.blockedAccounts,
+          };
+        }
+        errorCount++;
+        const message = e instanceof Error ? e.message : String(e);
+        const stack = e instanceof Error ? e.stack : undefined;
+        emit({ type: 'task-error', sourceId: task.sourceId, message, ...(stack && { stack }) });
+        return {
+          task,
+          result: {
+            inserted: 0,
+            skipped: 0,
+            insertedIds: [],
+            insertedTransactions: [],
+            changedTransactions: [],
+          },
+          status: 'error',
+          error: message,
+        };
+      }
+    };
+
+    let outcomes: TaskOutcome[];
+    if (concurrent) {
+      outcomes = await Promise.all(tasks.map(runTask));
+    } else {
+      outcomes = [];
+      for (const task of tasks) {
+        outcomes.push(await runTask(task));
+      }
+    }
+
+    const totalInserted = outcomes.reduce((sum, o) => sum + o.result.inserted, 0);
+    const totalSkipped = outcomes.reduce((sum, o) => sum + o.result.skipped, 0);
+    const sources: SourceRunRecord[] = outcomes.map(o => ({
+      sourceId: o.task.sourceId,
+      nickname: o.task.nickname,
+      sourceType: o.task.type,
+      status: o.status,
+      inserted: o.result.inserted,
+      skipped: o.result.skipped,
+      ...(o.error != null && { error: o.error }),
+      ...(o.blockedAccounts != null && { blockedAccounts: o.blockedAccounts }),
+    }));
+
+    emit({ type: 'run-complete', totalInserted, totalSkipped, errors: errorCount });
+
+    return {
+      id,
+      startedAt,
+      completedAt: new Date(),
+      totalInserted,
+      totalSkipped,
+      errorCount,
+      sources,
+    };
+  } finally {
+    _running = false;
+  }
+}

package/src/server/scrapers/__tests__/amex.test.ts

@@ -0,0 +1,142 @@
+import { describe, expect, it, vi } from 'vitest';
+import { PayloadValidationError } from '../../validate-payload.js';
+import { scrapeAmex } from '../amex.js';
+
+const CREDS = { id: 'src-1', ownerId: '123456789', password: 'pass', last6Digits: '123456' };
+
+const VALID_PAYLOAD = {
+  Header: { Status: '1', Message: null },
+  CardsTransactionsListBean: {
+    cardNumberList: ['010101'],
+    Index0: { '@AllCards': 'AllCards', CurrentCardTransactions: [] },
+  },
+};
+
+const DATE_FROM = new Date('2024-01-01');
+const DATE_TO = new Date('2024-01-31');
+const noop = () => {};
+
+vi.mock('@accounter/modern-poalim-scraper', () => ({
+  init: vi.fn(),
+}));
+
+async function getInitMock() {
+  const mod = await import('@accounter/modern-poalim-scraper');
+  return mod.init as ReturnType<typeof vi.fn>;
+}
+
+describe('scrapeAmex — happy path', () => {
+  it('resolves with an array of validated AmexPayloads', async () => {
+    const initMock = await getInitMock();
+    initMock.mockResolvedValue({
+      amex: vi.fn().mockResolvedValue({
+        getMonthTransactions: vi.fn().mockResolvedValue({ data: VALID_PAYLOAD, isValid: true }),
+      }),
+      close: vi.fn().mockResolvedValue(undefined),
+    });
+
+    const result = await scrapeAmex(CREDS, DATE_FROM, DATE_TO, noop);
+    expect(Array.isArray(result)).toBe(true);
+    expect(result).toHaveLength(1);
+    expect(result[0]!.Header.Status).toBe('1');
+  });
+
+  it('fetches one month per month in the date range', async () => {
+    const getMonthTransactions = vi
+      .fn()
+      .mockResolvedValue({ data: VALID_PAYLOAD, isValid: true });
+    const initMock = await getInitMock();
+    initMock.mockResolvedValue({
+      amex: vi.fn().mockResolvedValue({ getMonthTransactions }),
+      close: vi.fn().mockResolvedValue(undefined),
+    });
+
+    const threeMonthsFrom = new Date('2024-01-01');
+    const threeMonthsTo = new Date('2024-03-31');
+    await scrapeAmex(CREDS, threeMonthsFrom, threeMonthsTo, noop);
+
+    expect(getMonthTransactions).toHaveBeenCalledTimes(3);
+  });
+
+  it('skips months where data is null', async () => {
+    const initMock = await getInitMock();
+    initMock.mockResolvedValue({
+      amex: vi.fn().mockResolvedValue({
+        getMonthTransactions: vi.fn().mockResolvedValue({ data: null, isValid: null }),
+      }),
+      close: vi.fn().mockResolvedValue(undefined),
+    });
+
+    const result = await scrapeAmex(CREDS, DATE_FROM, DATE_TO, noop);
+    expect(result).toEqual([]);
+  });
+
+  it('calls close() even on success', async () => {
+    const close = vi.fn().mockResolvedValue(undefined);
+    const initMock = await getInitMock();
+    initMock.mockResolvedValue({
+      amex: vi.fn().mockResolvedValue({
+        getMonthTransactions: vi.fn().mockResolvedValue({ data: VALID_PAYLOAD, isValid: true }),
+      }),
+      close,
+    });
+
+    await scrapeAmex(CREDS, DATE_FROM, DATE_TO, noop);
+    expect(close).toHaveBeenCalledOnce();
+  });
+});
+
+describe('scrapeAmex — Header.Status check', () => {
+  it('throws when Header.Status is not "1"', async () => {
+    const initMock = await getInitMock();
+    initMock.mockResolvedValue({
+      amex: vi.fn().mockResolvedValue({
+        getMonthTransactions: vi.fn().mockResolvedValue({
+          data: { ...VALID_PAYLOAD, Header: { Status: '2', Message: 'Replace password' } },
+          isValid: true,
+        }),
+      }),
+      close: vi.fn().mockResolvedValue(undefined),
+    });
+
+    await expect(scrapeAmex(CREDS, DATE_FROM, DATE_TO, noop)).rejects.toThrow(
+      'login/password issue',
+    );
+  });
+});
+
+describe('scrapeAmex — invalid payload', () => {
+  it('throws PayloadValidationError when data fails schema validation', async () => {
+    const initMock = await getInitMock();
+    initMock.mockResolvedValue({
+      amex: vi.fn().mockResolvedValue({
+        getMonthTransactions: vi.fn().mockResolvedValue({
+          data: { Header: { Status: '1', Message: null }, unexpected: 'shape' },
+          isValid: true,
+        }),
+      }),
+      close: vi.fn().mockResolvedValue(undefined),
+    });
+
+    await expect(scrapeAmex(CREDS, DATE_FROM, DATE_TO, noop)).rejects.toBeInstanceOf(
+      PayloadValidationError,
+    );
+  });
+
+  it('calls close() even on validation error', async () => {
+    const close = vi.fn().mockResolvedValue(undefined);
+    const initMock = await getInitMock();
+    initMock.mockResolvedValue({
+      amex: vi.fn().mockResolvedValue({
+        getMonthTransactions: vi.fn().mockResolvedValue({
+          data: { Header: { Status: '1', Message: null }, unexpected: 'shape' },
+          isValid: true,
+        }),
+      }),
+      close,
+    });
+
+    await expect(scrapeAmex(CREDS, DATE_FROM, DATE_TO, noop)).rejects.toThrow();
+    expect(close).toHaveBeenCalledOnce();
+  });
+});

package/src/server/scrapers/__tests__/cal.test.ts

@@ -0,0 +1,135 @@
+import { describe, expect, it, vi } from 'vitest';
+import { PayloadValidationError } from '../../validate-payload.js';
+import { scrapeCal } from '../cal.js';
+
+const CREDS = { id: 'src-1', username: 'user', password: 'pass', last4Digits: '1234' };
+
+const DATE_FROM = new Date('2024-01-01');
+const DATE_TO = new Date('2024-01-31');
+const noop = () => {};
+
+vi.mock('@accounter/modern-poalim-scraper', () => ({
+  init: vi.fn(),
+}));
+
+async function getInitMock() {
+  const mod = await import('@accounter/modern-poalim-scraper');
+  return mod.init as ReturnType<typeof vi.fn>;
+}
+
+describe('scrapeCal — happy path', () => {
+  it('resolves with a validated CalPayload containing one entry per month', async () => {
+    const initMock = await getInitMock();
+    initMock.mockResolvedValue({
+      cal: vi.fn().mockResolvedValue({
+        getMonthTransactions: vi.fn().mockResolvedValue([]),
+      }),
+      close: vi.fn().mockResolvedValue(undefined),
+    });
+
+    const result = await scrapeCal(CREDS, DATE_FROM, DATE_TO, noop);
+    expect(Array.isArray(result)).toBe(true);
+    expect(result).toHaveLength(1);
+    expect(result[0]).toMatchObject({ card: '1234', month: '2024-01', transactions: [] });
+  });
+
+  it('fetches one month per month in the date range', async () => {
+    const getMonthTransactions = vi.fn().mockResolvedValue([]);
+    const initMock = await getInitMock();
+    initMock.mockResolvedValue({
+      cal: vi.fn().mockResolvedValue({ getMonthTransactions }),
+      close: vi.fn().mockResolvedValue(undefined),
+    });
+
+    const threeMonthsFrom = new Date('2024-01-01');
+    const threeMonthsTo = new Date('2024-03-31');
+    const result = await scrapeCal(CREDS, threeMonthsFrom, threeMonthsTo, noop);
+
+    expect(getMonthTransactions).toHaveBeenCalledTimes(3);
+    expect(result).toHaveLength(3);
+    expect(result.map(r => r.month)).toEqual(['2024-01', '2024-02', '2024-03']);
+  });
+
+  it('passes last4Digits and month correctly to getMonthTransactions', async () => {
+    const getMonthTransactions = vi.fn().mockResolvedValue([]);
+    const initMock = await getInitMock();
+    initMock.mockResolvedValue({
+      cal: vi.fn().mockResolvedValue({ getMonthTransactions }),
+      close: vi.fn().mockResolvedValue(undefined),
+    });
+
+    await scrapeCal(CREDS, DATE_FROM, DATE_TO, noop);
+
+    expect(getMonthTransactions).toHaveBeenCalledWith('1234', expect.any(Date));
+  });
+
+  it('calls close() even on success', async () => {
+    const close = vi.fn().mockResolvedValue(undefined);
+    const initMock = await getInitMock();
+    initMock.mockResolvedValue({
+      cal: vi.fn().mockResolvedValue({ getMonthTransactions: vi.fn().mockResolvedValue([]) }),
+      close,
+    });
+
+    await scrapeCal(CREDS, DATE_FROM, DATE_TO, noop);
+    expect(close).toHaveBeenCalledOnce();
+  });
+});
+
+describe('scrapeCal — transactions included', () => {
+  it('includes transactions returned by getMonthTransactions', async () => {
+    const mockTxn = {
+      trnIntId: 'txn-1',
+      merchantName: 'Shop',
+      trnPurchaseDate: '2024-01-15',
+      trnAmt: 100,
+      trnCurrencySymbol: 'ILS',
+      trnType: 'normal',
+      debCrdDate: '2024-02-01',
+      amtBeforeConvAndIndex: 100,
+      debCrdCurrencySymbol: 'ILS',
+    };
+    const initMock = await getInitMock();
+    initMock.mockResolvedValue({
+      cal: vi.fn().mockResolvedValue({
+        getMonthTransactions: vi.fn().mockResolvedValue([mockTxn]),
+      }),
+      close: vi.fn().mockResolvedValue(undefined),
+    });
+
+    const result = await scrapeCal(CREDS, DATE_FROM, DATE_TO, noop);
+    expect(result[0]!.transactions).toHaveLength(1);
+    expect(result[0]!.transactions[0]!.merchantName).toBe('Shop');
+  });
+});
+
+describe('scrapeCal — invalid payload', () => {
+  it('throws PayloadValidationError when a transaction fails schema validation', async () => {
+    const initMock = await getInitMock();
+    initMock.mockResolvedValue({
+      cal: vi.fn().mockResolvedValue({
+        // Return a transaction missing required fields
+        getMonthTransactions: vi.fn().mockResolvedValue([{ trnIntId: 'x' }]),
+      }),
+      close: vi.fn().mockResolvedValue(undefined),
+    });
+
+    await expect(scrapeCal(CREDS, DATE_FROM, DATE_TO, noop)).rejects.toBeInstanceOf(
+      PayloadValidationError,
+    );
+  });
+
+  it('calls close() even on validation error', async () => {
+    const close = vi.fn().mockResolvedValue(undefined);
+    const initMock = await getInitMock();
+    initMock.mockResolvedValue({
+      cal: vi.fn().mockResolvedValue({
+        getMonthTransactions: vi.fn().mockResolvedValue([{ trnIntId: 'x' }]),
+      }),
+      close,
+    });
+
+    await expect(scrapeCal(CREDS, DATE_FROM, DATE_TO, noop)).rejects.toThrow();
+    expect(close).toHaveBeenCalledOnce();
+  });
+});