@accounter/scraper-app 0.0.1

package/src/server/__tests__/validate-payload.test.ts

@@ -0,0 +1,193 @@
+import { describe, expect, it, afterEach } from 'vitest';
+import { PayloadValidationError, validatePayload } from '../validate-payload.js';
+import { _resetRunState, startRun, type ScrapeTask } from '../scrape-runner.js';
+import type { ServerMessage } from '../../shared/ws-protocol.js';
+
+describe('validatePayload — valid fixtures', () => {
+  it('accepts a minimal poalim-ils payload', () => {
+    const result = validatePayload('poalim-ils', {
+      transactions: [
+        {
+          activityDescription: 'Credit',
+          activityTypeCode: 1,
+          eventAmount: 100,
+          eventDate: 20240101,
+          serialNumber: 1,
+          transactionType: 'REGULAR',
+          currentBalance: 5000,
+          referenceNumber: 12345,
+        },
+      ],
+      retrievalTransactionData: { accountNumber: 100000, branchNumber: 600, bankNumber: 12 },
+    });
+    expect(result.transactions).toHaveLength(1);
+  });
+
+  it('accepts a poalim-ils payload with extra fields (passthrough)', () => {
+    const result = validatePayload('poalim-ils', {
+      transactions: [],
+      retrievalTransactionData: { accountNumber: 1, branchNumber: 2, bankNumber: 12 },
+      unknownField: 'should be kept',
+    });
+    expect((result as Record<string, unknown>)['unknownField']).toBe('should be kept');
+  });
+
+  it('accepts a minimal poalim-foreign payload', () => {
+    const result = validatePayload('poalim-foreign', {
+      balancesAndLimitsDataList: [
+        {
+          currencySwiftCode: 'USD',
+          currencyCode: 1,
+          transactions: [],
+        },
+      ],
+    });
+    expect(result.balancesAndLimitsDataList).toHaveLength(1);
+  });
+
+  it('accepts a minimal poalim-swift payload', () => {
+    const result = validatePayload('poalim-swift', { swiftsList: [] });
+    expect(result.swiftsList).toHaveLength(0);
+  });
+
+  it('accepts a minimal isracard payload', () => {
+    const result = validatePayload('isracard', {
+      Header: { Status: '1', Message: null },
+      CardsTransactionsListBean: {
+        cardNumberList: ['012345'],
+        Index0: { '@AllCards': 'AllCards', CurrentCardTransactions: [] },
+      },
+    });
+    expect(result.Header.Status).toBe('1');
+  });
+
+  it('accepts amex with the same shape as isracard', () => {
+    const result = validatePayload('amex', {
+      Header: { Status: '1', Message: null },
+      CardsTransactionsListBean: {
+        cardNumberList: ['343434'],
+        Index0: { '@AllCards': 'AllCards', CurrentCardTransactions: [] },
+      },
+    });
+    expect(result.Header.Status).toBe('1');
+  });
+
+  it('accepts a minimal cal payload', () => {
+    const result = validatePayload('cal', [
+      { card: '1234', month: '2024-01', transactions: [] },
+    ]);
+    expect(result).toHaveLength(1);
+    expect(result[0]!.card).toBe('1234');
+  });
+
+  it('accepts a minimal discount payload', () => {
+    const result = validatePayload('discount', [
+      { accountNumber: 'ACC-001', month: '2024-01', balance: 5000, transactions: [] },
+    ]);
+    expect(result).toHaveLength(1);
+    expect(result[0]!.accountNumber).toBe('ACC-001');
+    expect(result[0]!.balance).toBe(5000);
+  });
+
+  it('accepts a minimal max payload', () => {
+    const result = validatePayload('max', [{ accountNumber: '1234', txns: [] }]);
+    expect(result).toHaveLength(1);
+    expect(result[0]!.accountNumber).toBe('1234');
+  });
+
+  it('accepts an empty max payload', () => {
+    const result = validatePayload('max', []);
+    expect(result).toEqual([]);
+  });
+
+  it('accepts a currency-rates payload', () => {
+    const result = validatePayload('currency-rates', [
+      { date: '2024-01-01', currency: 'USD', rate: 3.712 },
+    ]);
+    expect(result).toHaveLength(1);
+    expect(result[0]!.currency).toBe('USD');
+  });
+});
+
+describe('validatePayload — invalid fixtures', () => {
+  it('throws PayloadValidationError for wrong transactions type in poalim-ils', () => {
+    expect(() =>
+      validatePayload('poalim-ils', {
+        transactions: 'not-an-array',
+        retrievalTransactionData: { accountNumber: 1, branchNumber: 2, bankNumber: 12 },
+      }),
+    ).toThrow(PayloadValidationError);
+  });
+
+  it('error message includes the payload type', () => {
+    try {
+      validatePayload('poalim-ils', { transactions: 42 });
+    } catch (e) {
+      expect(e).toBeInstanceOf(PayloadValidationError);
+      expect((e as PayloadValidationError).message).toContain('poalim-ils');
+      expect((e as PayloadValidationError).payloadType).toBe('poalim-ils');
+    }
+  });
+
+  it('throws for missing required field in discount payload', () => {
+    expect(() =>
+      validatePayload('discount', [
+        { accountNumber: 'ACC-001', month: '2024-01' }, // missing balance and transactions
+      ]),
+    ).toThrow(PayloadValidationError);
+  });
+
+  it('throws for invalid transaction type in poalim-ils', () => {
+    expect(() =>
+      validatePayload('poalim-ils', {
+        transactions: [
+          {
+            activityDescription: 'X',
+            activityTypeCode: 1,
+            eventAmount: 100,
+            eventDate: 20240101,
+            serialNumber: 1,
+            transactionType: 'INVALID_TYPE',
+            currentBalance: 5000,
+            referenceNumber: 1,
+          },
+        ],
+        retrievalTransactionData: { accountNumber: 1, branchNumber: 2, bankNumber: 12 },
+      }),
+    ).toThrow(PayloadValidationError);
+  });
+
+  it('throws for wrong currency in currency-rates', () => {
+    expect(() =>
+      validatePayload('currency-rates', [{ date: '2024-01-01', currency: 'XYZ', rate: 1.0 }]),
+    ).toThrow(PayloadValidationError);
+  });
+});
+
+describe('runner integration — task-error on PayloadValidationError', () => {
+  afterEach(() => {
+    _resetRunState();
+  });
+
+  it('emits task-error (not a crash) when validatePayload throws inside run()', async () => {
+    const events: ServerMessage[] = [];
+
+    const task: ScrapeTask = {
+      sourceId: 'bad-src',
+      nickname: 'bad-src',
+      type: 'poalim',
+      run: async () => {
+        // Deliberately pass invalid data to trigger PayloadValidationError
+        validatePayload('poalim-ils', { transactions: 'not-an-array' });
+        return { inserted: 0, skipped: 0, insertedIds: [] };
+      },
+    };
+
+    await startRun([task], false, msg => events.push(msg));
+
+    const taskError = events.find(e => e.type === 'task-error');
+    expect(taskError).toBeTruthy();
+    expect((taskError as { sourceId: string }).sourceId).toBe('bad-src');
+    expect(events.at(-1)).toMatchObject({ type: 'run-complete', totalInserted: 0, totalSkipped: 0 });
+  });
+});

package/src/server/__tests__/vault-routes.test.ts

@@ -0,0 +1,174 @@
+import { randomBytes } from 'node:crypto';
+import { rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import Fastify, { type FastifyInstance } from 'fastify';
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import { defaultVault, saveVaultFile } from '../vault.js';
+import { lockVault } from '../vault-store.js';
+import { registerVaultRoutes } from '../vault-routes.js';
+
+const PASSWORD = 'test-password-123';
+
+function makeTmpPath() {
+  return join(tmpdir(), `vault-test-${randomBytes(4).toString('hex')}.vault`);
+}
+
+async function buildApp(vaultPath: string): Promise<FastifyInstance> {
+  process.env['VAULT_PATH'] = vaultPath;
+  const app = Fastify();
+  await registerVaultRoutes(app);
+  await app.ready();
+  return app;
+}
+
+// ── Tests where the vault file does NOT yet exist ─────────────────────────────
+
+describe('GET /api/vault/status', () => {
+  let vaultPath: string;
+  let app: FastifyInstance;
+
+  beforeEach(async () => {
+    vaultPath = makeTmpPath();
+    app = await buildApp(vaultPath);
+  });
+
+  afterEach(async () => {
+    lockVault();
+    await app.close();
+    await rm(vaultPath, { force: true });
+    delete process.env['VAULT_PATH'];
+  });
+
+  it('returns locked=true and hasFile=false when no vault file exists', async () => {
+    const res = await app.inject({ method: 'GET', url: '/api/vault/status' });
+    expect(res.statusCode).toBe(200);
+    expect(res.json()).toEqual({ locked: true, hasFile: false });
+  });
+});
+
+// ── Tests for POST /api/vault/create ─────────────────────────────────────────
+
+describe('POST /api/vault/create', () => {
+  let vaultPath: string;
+  let app: FastifyInstance;
+
+  beforeEach(async () => {
+    vaultPath = makeTmpPath();
+    app = await buildApp(vaultPath);
+  });
+
+  afterEach(async () => {
+    lockVault();
+    await app.close();
+    await rm(vaultPath, { force: true });
+    delete process.env['VAULT_PATH'];
+  });
+
+  it('creates a vault and returns 201', async () => {
+    const res = await app.inject({
+      method: 'POST',
+      url: '/api/vault/create',
+      payload: { password: PASSWORD, serverUrl: 'http://localhost:4000/graphql', apiKey: 'key1' },
+    });
+    expect(res.statusCode).toBe(201);
+    expect(res.json()).toEqual({ ok: true });
+  });
+
+  it('GET /api/vault/status after create returns locked=false and hasFile=true', async () => {
+    await app.inject({
+      method: 'POST',
+      url: '/api/vault/create',
+      payload: { password: PASSWORD, serverUrl: 'http://localhost:4000/graphql', apiKey: 'key1' },
+    });
+    const res = await app.inject({ method: 'GET', url: '/api/vault/status' });
+    expect(res.json()).toEqual({ locked: false, hasFile: true });
+  });
+
+  it('returns 409 when vault file already exists', async () => {
+    await app.inject({
+      method: 'POST',
+      url: '/api/vault/create',
+      payload: { password: PASSWORD, serverUrl: 'http://localhost:4000/graphql', apiKey: 'key1' },
+    });
+    const res = await app.inject({
+      method: 'POST',
+      url: '/api/vault/create',
+      payload: { password: PASSWORD, serverUrl: 'http://localhost:4000/graphql', apiKey: 'key1' },
+    });
+    expect(res.statusCode).toBe(409);
+  });
+});
+
+// ── Tests where the vault file ALREADY exists ─────────────────────────────────
+
+describe('GET /api/vault/status', () => {
+  let vaultPath: string;
+  let app: FastifyInstance;
+
+  beforeEach(async () => {
+    vaultPath = makeTmpPath();
+    await saveVaultFile(vaultPath, defaultVault(), PASSWORD);
+    app = await buildApp(vaultPath);
+  });
+
+  afterEach(async () => {
+    lockVault();
+    await app.close();
+    await rm(vaultPath, { force: true });
+    delete process.env['VAULT_PATH'];
+  });
+
+  it('returns locked=true and hasFile=true before unlock', async () => {
+    const res = await app.inject({ method: 'GET', url: '/api/vault/status' });
+    expect(res.statusCode).toBe(200);
+    expect(res.json()).toEqual({ locked: true, hasFile: true });
+  });
+
+  it('returns locked=false after successful unlock', async () => {
+    await app.inject({
+      method: 'POST',
+      url: '/api/vault/unlock',
+      payload: { password: PASSWORD },
+    });
+    const res = await app.inject({ method: 'GET', url: '/api/vault/status' });
+    expect(res.json()).toMatchObject({ locked: false, hasFile: true });
+  });
+});
+
+describe('POST /api/vault/unlock', () => {
+  let vaultPath: string;
+  let app: FastifyInstance;
+
+  beforeEach(async () => {
+    vaultPath = makeTmpPath();
+    await saveVaultFile(vaultPath, defaultVault(), PASSWORD);
+    app = await buildApp(vaultPath);
+  });
+
+  afterEach(async () => {
+    lockVault();
+    await app.close();
+    await rm(vaultPath, { force: true });
+    delete process.env['VAULT_PATH'];
+  });
+
+  it('returns ok with correct password', async () => {
+    const res = await app.inject({
+      method: 'POST',
+      url: '/api/vault/unlock',
+      payload: { password: PASSWORD },
+    });
+    expect(res.statusCode).toBe(200);
+    expect(res.json()).toEqual({ ok: true });
+  });
+
+  it('returns 401 with wrong password', async () => {
+    const res = await app.inject({
+      method: 'POST',
+      url: '/api/vault/unlock',
+      payload: { password: 'wrong-password' },
+    });
+    expect(res.statusCode).toBe(401);
+  });
+});

package/src/server/__tests__/vault.test.ts

@@ -0,0 +1,33 @@
+import { describe, expect, it } from 'vitest';
+import { decryptVault, defaultVault, encryptVault, VaultSchema } from '../vault.js';
+
+describe('vault', () => {
+  it('encrypt→decrypt round-trip returns original vault', async () => {
+    const vault = defaultVault();
+    vault.poalimAccounts.push({ id: 'test-id-1', userCode: 'user1', password: 'pass1' });
+    const blob = await encryptVault(vault, 'my-password');
+    expect(await decryptVault(blob, 'my-password')).toEqual(vault);
+  });
+
+  it('wrong password returns null without throwing', async () => {
+    const blob = await encryptVault(defaultVault(), 'correct-password');
+    await expect(decryptVault(blob, 'wrong-password')).resolves.toBeNull();
+  });
+
+  it('tampered ciphertext returns null', async () => {
+    const blob = await encryptVault(defaultVault(), 'password');
+    const buf = Buffer.from(blob, 'base64');
+    buf[buf.length - 1] ^= 0xff;
+    expect(await decryptVault(buf.toString('base64'), 'password')).toBeNull();
+  });
+
+  it('VaultSchema rejects missing required fields', () => {
+    expect(() =>
+      VaultSchema.parse({ poalimAccounts: [{ password: 'no-user-code' }] }),
+    ).toThrow();
+  });
+
+  it('defaultVault() passes VaultSchema.parse', () => {
+    expect(() => VaultSchema.parse(defaultVault())).not.toThrow();
+  });
+});

package/src/server/__tests__/websocket.test.ts

@@ -0,0 +1,151 @@
+import '@fastify/websocket'; // type augmentation for app.injectWS
+import Fastify, { type FastifyInstance } from 'fastify';
+import type { WebSocket } from 'ws';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { registerWebSocketRoute } from '../websocket.js';
+
+const STUB_VAULT = {
+  poalimAccounts: [{ id: 'src-1', userCode: 'u', password: 'p' }],
+  discountAccounts: [],
+  isracardAccounts: [],
+  amexAccounts: [],
+  calAccounts: [],
+  maxAccounts: [],
+  accountRecords: [],
+  settings: { showBrowser: false, fetchBankOfIsraelRates: true, concurrentScraping: false },
+};
+
+vi.mock('../vault-store.js', () => ({ isLocked: () => false, getVault: () => STUB_VAULT }));
+vi.mock('../scrape-runner.js', () => ({ startRun: vi.fn().mockResolvedValue(undefined) }));
+
+let app: FastifyInstance;
+
+// Buffered message queue: onMessage collects arrivals; next() dequeues or waits.
+// Set up via onInit so the listener is active before the connection opens —
+// guarantees no message is ever missed regardless of event-loop ordering.
+function makeClient() {
+  const queue: unknown[] = [];
+  const waiters: Array<(m: unknown) => void> = [];
+
+  const onMessage = (data: { toString(): string }) => {
+    const msg = JSON.parse(data.toString()) as unknown;
+    const resolve = waiters.shift();
+    if (resolve) resolve(msg);
+    else queue.push(msg);
+  };
+
+  const next = (ms = 2000): Promise<unknown> => {
+    if (queue.length > 0) return Promise.resolve(queue.shift());
+    return new Promise((res, rej) => {
+      const t = setTimeout(() => {
+        const i = waiters.indexOf(res);
+        if (i !== -1) waiters.splice(i, 1);
+        rej(new Error('timeout waiting for WS message'));
+      }, ms);
+      waiters.push(m => {
+        clearTimeout(t);
+        res(m);
+      });
+    });
+  };
+
+  return { onMessage, next };
+}
+
+async function openSocket(
+  client: ReturnType<typeof makeClient>,
+): Promise<WebSocket> {
+  return app.injectWS('/ws', undefined, {
+    onInit: (w: WebSocket) => w.on('message', client.onMessage),
+  });
+}
+
+beforeEach(async () => {
+  app = Fastify({ logger: false });
+  await registerWebSocketRoute(app);
+  await app.ready();
+});
+
+afterEach(async () => {
+  await app.close();
+});
+
+describe('WebSocket /ws', () => {
+  it('sends connected ack immediately on connection', async () => {
+    const client = makeClient();
+    const ws = await openSocket(client);
+    expect(await client.next()).toEqual({ type: 'connected' });
+    ws.close();
+  });
+
+  it('responds to ping with pong', async () => {
+    const client = makeClient();
+    const ws = await openSocket(client);
+    await client.next(); // consume connected
+    ws.send(JSON.stringify({ type: 'ping' }));
+    expect(await client.next()).toEqual({ type: 'pong' });
+    ws.close();
+  });
+
+  it('sends error reply for unknown message type', async () => {
+    const client = makeClient();
+    const ws = await openSocket(client);
+    await client.next(); // consume connected
+
+    ws.send(JSON.stringify({ type: 'totally-unknown', payload: 42 }));
+
+    expect(await client.next()).toEqual({ type: 'error', message: 'Unknown message type' });
+    ws.close();
+  });
+
+  it('server stays alive after unknown message type', async () => {
+    const client = makeClient();
+    const ws = await openSocket(client);
+    await client.next(); // consume connected
+
+    ws.send(JSON.stringify({ type: 'totally-unknown', payload: 42 }));
+    await client.next(); // consume error reply
+
+    ws.send(JSON.stringify({ type: 'ping' }));
+    expect(await client.next()).toEqual({ type: 'pong' });
+    ws.close();
+  });
+
+  it('accepts run-start message without crashing (stub handler)', async () => {
+    const client = makeClient();
+    const ws = await openSocket(client);
+    await client.next(); // consume connected
+
+    ws.send(JSON.stringify({ type: 'run-start', sourceIds: ['src-1'] }));
+    ws.send(JSON.stringify({ type: 'ping' }));
+
+    // pong proves the server processed run-start without crashing
+    expect(await client.next()).toEqual({ type: 'pong' });
+    ws.close();
+  });
+
+  it('does not crash on malformed JSON — server stays alive', async () => {
+    const client = makeClient();
+    const ws = await openSocket(client);
+    await client.next();
+
+    ws.send('this is {{ not valid json');
+    ws.send(JSON.stringify({ type: 'ping' }));
+
+    expect(await client.next()).toEqual({ type: 'pong' });
+    ws.close();
+  });
+
+  it('handles multiple concurrent connections independently', async () => {
+    const c1 = makeClient();
+    const c2 = makeClient();
+    const ws1 = await openSocket(c1);
+    const ws2 = await openSocket(c2);
+
+    expect(await c1.next()).toEqual({ type: 'connected' });
+    expect(await c2.next()).toEqual({ type: 'connected' });
+
+    ws1.close();
+    ws2.close();
+  });
+});

package/src/server/account-discovery.ts

@@ -0,0 +1,49 @@
+import { randomUUID } from 'node:crypto';
+import {
+  extractAccountIdentifiers,
+  type SourceType,
+  type ValidatedPayload,
+} from './check-accounts.js';
+import { getVault, updateVault } from './vault-store.js';
+import type { AccountRecord } from './vault.js';
+
+export async function registerDiscoveredAccounts(
+  sourceType: SourceType,
+  sourceId: string,
+  payloads: ValidatedPayload[],
+): Promise<void> {
+  const existing = getVault().accountRecords;
+  const newRecords: AccountRecord[] = [];
+
+  for (const payload of payloads) {
+    for (const accountNumber of extractAccountIdentifiers(sourceType, payload)) {
+      const alreadyKnown = existing.some(
+        r => r.sourceType === sourceType && r.accountNumber === accountNumber,
+      );
+      if (
+        !alreadyKnown &&
+        !newRecords.some(
+          r =>
+            r.sourceType === sourceType &&
+            r.sourceId === sourceId &&
+            r.accountNumber === accountNumber,
+        )
+      ) {
+        newRecords.push({
+          id: randomUUID(),
+          sourceId,
+          sourceType,
+          accountNumber,
+          status: 'pending' as const,
+        });
+      }
+    }
+  }
+
+  if (newRecords.length > 0) {
+    await updateVault(v => ({
+      ...v,
+      accountRecords: [...v.accountRecords, ...newRecords],
+    }));
+  }
+}

package/src/server/accounts-routes.ts

@@ -0,0 +1,74 @@
+import type { FastifyInstance } from 'fastify';
+import { getVault, isLocked, updateVault } from './vault-store.js';
+
+type StatusBody = { status: 'accepted' | 'ignored' };
+
+function guardLocked(reply: { status(code: number): { send(body: unknown): unknown } }) {
+  if (isLocked()) return reply.status(401).send({ error: 'vault-locked' });
+  return null;
+}
+
+export async function registerAccountsRoutes(app: FastifyInstance): Promise<void> {
+  app.get('/api/vault/accounts', async (_req, reply) => {
+    const blocked = guardLocked(reply);
+    if (blocked) return blocked;
+    const vault = getVault();
+    const { accountRecords } = vault;
+    return accountRecords.map(r => ({
+      ...r,
+      nickname: (
+        vault[`${r.sourceType}Accounts` as keyof typeof vault] as {
+          id: string;
+          nickname?: string;
+        }[]
+      ).find(a => a.id === r.sourceId)?.nickname,
+    }));
+  });
+
+  app.put<{ Params: { id: string }; Body: StatusBody }>(
+    '/api/vault/accounts/:id',
+    async (req, reply) => {
+      const blocked = guardLocked(reply);
+      if (blocked) return blocked;
+
+      const { id } = req.params;
+      const { status } = req.body;
+
+      if (status !== 'accepted' && status !== 'ignored') {
+        return reply.status(400).send({ error: 'status must be accepted or ignored' });
+      }
+
+      const vault = getVault();
+      if (!vault.accountRecords.some(a => a.id === id)) {
+        return reply.status(404).send({ error: 'not-found' });
+      }
+
+      await updateVault(v => {
+        const idx = v.accountRecords.findIndex(a => a.id === id);
+        if (idx === -1) return v;
+        const accounts = [...v.accountRecords];
+        accounts[idx] = { ...accounts[idx], status };
+        return { ...v, accountRecords: accounts };
+      });
+
+      return getVault().accountRecords;
+    },
+  );
+
+  app.delete<{ Params: { id: string } }>('/api/vault/accounts/:id', async (req, reply) => {
+    const blocked = guardLocked(reply);
+    if (blocked) return blocked;
+
+    const { id } = req.params;
+    if (!getVault().accountRecords.some(a => a.id === id)) {
+      return reply.status(404).send({ error: 'not-found' });
+    }
+
+    await updateVault(v => ({
+      ...v,
+      accountRecords: v.accountRecords.filter(a => a.id !== id),
+    }));
+
+    return getVault().accountRecords;
+  });
+}