@accounter/scraper-app 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. package/README.md +90 -0
  2. package/docs/plan.md +76 -0
  3. package/index.html +12 -0
  4. package/package.json +40 -0
  5. package/src/env.template +2 -0
  6. package/src/server/__tests__/accounts-routes.test.ts +133 -0
  7. package/src/server/__tests__/check-accounts.test.ts +305 -0
  8. package/src/server/__tests__/filter-payload.test.ts +193 -0
  9. package/src/server/__tests__/graphql-client.integration.test.ts +98 -0
  10. package/src/server/__tests__/graphql-client.test.ts +508 -0
  11. package/src/server/__tests__/healthz.test.ts +22 -0
  12. package/src/server/__tests__/history.test.ts +111 -0
  13. package/src/server/__tests__/otp-manager.test.ts +132 -0
  14. package/src/server/__tests__/scrape-runner.test.ts +144 -0
  15. package/src/server/__tests__/settings-routes.test.ts +117 -0
  16. package/src/server/__tests__/sources-routes.test.ts +149 -0
  17. package/src/server/__tests__/validate-payload.test.ts +193 -0
  18. package/src/server/__tests__/vault-routes.test.ts +174 -0
  19. package/src/server/__tests__/vault.test.ts +33 -0
  20. package/src/server/__tests__/websocket.test.ts +151 -0
  21. package/src/server/account-discovery.ts +49 -0
  22. package/src/server/accounts-routes.ts +74 -0
  23. package/src/server/check-accounts.ts +79 -0
  24. package/src/server/filter-payload.ts +145 -0
  25. package/src/server/graphql/client.ts +103 -0
  26. package/src/server/graphql/mutations.ts +518 -0
  27. package/src/server/history-routes.ts +11 -0
  28. package/src/server/history.ts +53 -0
  29. package/src/server/index.ts +40 -0
  30. package/src/server/otp-manager.ts +63 -0
  31. package/src/server/payload-schemas/amex.schema.ts +2 -0
  32. package/src/server/payload-schemas/cal.schema.ts +27 -0
  33. package/src/server/payload-schemas/currency-rates.schema.ts +11 -0
  34. package/src/server/payload-schemas/discount.schema.ts +26 -0
  35. package/src/server/payload-schemas/isracard.schema.ts +58 -0
  36. package/src/server/payload-schemas/max.schema.ts +27 -0
  37. package/src/server/payload-schemas/poalim-foreign.schema.ts +30 -0
  38. package/src/server/payload-schemas/poalim-ils.schema.ts +31 -0
  39. package/src/server/payload-schemas/poalim-swift.schema.ts +21 -0
  40. package/src/server/scrape-runner.ts +165 -0
  41. package/src/server/scrapers/__tests__/amex.test.ts +142 -0
  42. package/src/server/scrapers/__tests__/cal.test.ts +135 -0
  43. package/src/server/scrapers/__tests__/currency-rates.test.ts +105 -0
  44. package/src/server/scrapers/__tests__/discount.test.ts +160 -0
  45. package/src/server/scrapers/__tests__/isracard.test.ts +142 -0
  46. package/src/server/scrapers/__tests__/max.test.ts +115 -0
  47. package/src/server/scrapers/__tests__/poalim.test.ts +154 -0
  48. package/src/server/scrapers/amex.ts +63 -0
  49. package/src/server/scrapers/cal.ts +56 -0
  50. package/src/server/scrapers/currency-rates.ts +64 -0
  51. package/src/server/scrapers/discount.ts +62 -0
  52. package/src/server/scrapers/isracard.ts +68 -0
  53. package/src/server/scrapers/max.ts +32 -0
  54. package/src/server/scrapers/poalim.ts +103 -0
  55. package/src/server/settings-routes.ts +27 -0
  56. package/src/server/sources-routes.ts +182 -0
  57. package/src/server/validate-payload.ts +74 -0
  58. package/src/server/vault-routes.ts +99 -0
  59. package/src/server/vault-store.ts +42 -0
  60. package/src/server/vault.ts +216 -0
  61. package/src/server/websocket.ts +454 -0
  62. package/src/shared/source-types.ts +10 -0
  63. package/src/shared/types.ts +20 -0
  64. package/src/shared/ws-protocol.ts +177 -0
  65. package/src/test-setup.ts +6 -0
  66. package/src/ui/__tests__/accounts-tab.test.tsx +134 -0
  67. package/src/ui/__tests__/config.test.tsx +99 -0
  68. package/src/ui/__tests__/history.test.tsx +94 -0
  69. package/src/ui/__tests__/run.test.tsx +195 -0
  70. package/src/ui/__tests__/settings-tab.test.tsx +79 -0
  71. package/src/ui/__tests__/sources-tab.test.tsx +139 -0
  72. package/src/ui/__tests__/vault-setup.test.tsx +105 -0
  73. package/src/ui/__tests__/vault-unlock.test.tsx +78 -0
  74. package/src/ui/app.tsx +109 -0
  75. package/src/ui/components/error-boundary.tsx +54 -0
  76. package/src/ui/components/otp-modal.tsx +82 -0
  77. package/src/ui/components/skeleton.tsx +58 -0
  78. package/src/ui/components/task-row.tsx +241 -0
  79. package/src/ui/contexts/vault-context.tsx +77 -0
  80. package/src/ui/lib/api.ts +117 -0
  81. package/src/ui/lib/ws.ts +137 -0
  82. package/src/ui/main.tsx +9 -0
  83. package/src/ui/screens/config/accounts-tab.tsx +185 -0
  84. package/src/ui/screens/config/config.tsx +163 -0
  85. package/src/ui/screens/config/settings-tab.tsx +167 -0
  86. package/src/ui/screens/config/source-forms.tsx +518 -0
  87. package/src/ui/screens/config/source-types.ts +91 -0
  88. package/src/ui/screens/config/sources-tab.tsx +176 -0
  89. package/src/ui/screens/history.tsx +234 -0
  90. package/src/ui/screens/run.tsx +266 -0
  91. package/src/ui/screens/vault-setup.tsx +120 -0
  92. package/src/ui/screens/vault-unlock.tsx +38 -0
  93. package/tsconfig.json +15 -0
  94. package/tsup.config.ts +10 -0
  95. package/vite.config.ts +24 -0
  96. package/vitest.config.ts +7 -0
package/README.md ADDED
@@ -0,0 +1,90 @@
1
+ # Scraper App
2
+
3
+ A local web app that scrapes Israeli bank accounts and uploads transactions to your Accounter
4
+ server.
5
+
6
+ ## Prerequisites
7
+
8
+ - **Node.js** v22 or later
9
+ - A running [Accounter](../../README.md) server with a scraper API key configured
10
+ - Yarn Berry (v4) — the repo uses Yarn workspaces
11
+
12
+ ## How to start
13
+
14
+ ### Development mode
15
+
16
+ Run the Fastify server (serves the UI and handles scraping):
17
+
18
+ ```bash
19
+ yarn workspace @accounter-helper/scraper-app dev:server
20
+ ```
21
+
22
+ Then open [http://localhost:4001](http://localhost:4001) in your browser.
23
+
24
+ > The default port is `4001`. Override it with the `PORT` environment variable (see below).
25
+
26
+ ### Build for production
27
+
28
+ ```bash
29
+ yarn workspace @accounter-helper/scraper-app build
30
+ ```
31
+
32
+ This produces:
33
+
34
+ - `dist/` — compiled Fastify server (run with `node dist/index.js`)
35
+ - `dist/ui/` — compiled React SPA (served statically by the server)
36
+
37
+ To run the production build:
38
+
39
+ ```bash
40
+ node packages/scraper-app/dist/index.js
41
+ ```
42
+
43
+ ## First-run vault setup
44
+
45
+ 1. Open the app in your browser. You will see the **Vault Setup** wizard.
46
+ 2. **Step 1 — Choose password**: Enter and confirm a master password. This encrypts all stored
47
+ credentials.
48
+ 3. **Step 2 — Server connection**: Enter your Accounter server URL (e.g.
49
+ `http://localhost:4000/graphql`) and your scraper API key.
50
+ 4. **Step 3 — Confirm**: Review and click **Create vault**. The vault is saved locally as an
51
+ encrypted file.
52
+ 5. You will be taken to the main app. Navigate to **Config → Credentials** to add bank sources.
53
+
54
+ On subsequent visits, you will be prompted to unlock the vault with your master password.
55
+
56
+ ## Adding bank sources
57
+
58
+ 1. Go to **Config → Credentials**.
59
+ 2. Select a source type from the dropdown and click **Add source**.
60
+ 3. Fill in the credentials form for your bank (username/password, OTP seed, etc.).
61
+ 4. Click **Save**. The source appears in the list.
62
+
63
+ ## Running a scrape
64
+
65
+ 1. Go to the **Run** tab.
66
+ 2. Select the sources you want to scrape (all are selected by default).
67
+ 3. Set the date range — either "Last N months" or a custom from/to range.
68
+ 4. Click **Run**. The scraper runs in the background; task rows show live status.
69
+ 5. If a Poalim account requires an OTP, a dialog will appear — enter the code and submit.
70
+ 6. When the run completes, a summary shows total new / skipped transactions.
71
+
72
+ ## Environment variables
73
+
74
+ | Variable | Default | Description |
75
+ | ------------ | -------- | --------------------------------------- |
76
+ | `PORT` | `4001` | HTTP port the Fastify server listens on |
77
+ | `VAULT_PATH` | `.vault` | Path to the encrypted vault file |
78
+
79
+ Example:
80
+
81
+ ```bash
82
+ PORT=8080 VAULT_PATH=/data/my.vault node dist/index.js
83
+ ```
84
+
85
+ ## Vault backup
86
+
87
+ The vault file (default: `.vault` in the working directory) contains all encrypted credentials.
88
+ **Back up this file** — if lost, you will need to re-enter all credentials.
89
+
90
+ The vault file path is also shown in **Config → Settings** with a one-click copy button.
package/docs/plan.md ADDED
@@ -0,0 +1,76 @@
1
+ # Plan: Auto-register discovered accounts into vault.accountRecords
2
+
3
+ ## TL;DR
4
+
5
+ When a scrape completes successfully, account identifiers extracted from payloads should be
6
+ auto-registered in `vault.accountRecords` as `pending` if not already known. `checkAccounts` should
7
+ then treat `pending` as blocking (same as unknown today), prompting the user to classify before
8
+ upload proceeds.
9
+
10
+ ## Steps
11
+
12
+ ### Phase 1 — Fix `checkAccounts` to treat `pending` as blocking (1 file)
13
+
14
+ - In `check-accounts.ts`, the `else` branch in `checkAccounts` currently accepts both `'accepted'`
15
+ and `'pending'`. Split it into explicit `accepted` vs. `pending`→`unknown` so pending accounts
16
+ block the task.
17
+
18
+ ### Phase 2 — New `registerDiscoveredAccounts` helper (new file or check-accounts.ts)
19
+
20
+ - Create a function `registerDiscoveredAccounts(sourceType, sourceId, payloads, updateVaultFn)`:
21
+ 1. Call `extractAccountIdentifiers` for each payload to collect all identifiers.
22
+ 2. Read current `vault.accountRecords` via `getVault()`.
23
+ 3. For each identifier not already present (by `sourceType + accountNumber`), create a new
24
+ `AccountRecord` with a `crypto.randomUUID()` id, the sourceId, sourceType, accountNumber, and
25
+ `status: 'pending'`.
26
+ 4. If any new records were created, call `updateVault` to persist them.
27
+ - This can live in a new `account-discovery.ts` file to keep check-accounts.ts read-only.
28
+
29
+ ### Phase 3 — Integrate into `websocket.ts` (2 call sites)
30
+
31
+ **Poalim path** (around line 81):
32
+
33
+ - After `filteredIls` is built, before `checkAccounts`, call
34
+ `registerDiscoveredAccounts('poalim', src.id, filteredIls)`.
35
+ - Re-read `vault.accountRecords` via `getVault().accountRecords` for the `checkAccounts` call (the
36
+ closure `vault` is stale after `updateVault`).
37
+
38
+ **Non-Poalim path** (around line 194):
39
+
40
+ - After filtering payloads and before `checkAccounts(src.type, payloads[0]!, ...)`, call
41
+ `registerDiscoveredAccounts(src.type, src.id, payloads)`.
42
+ - Same stale-vault concern: use `getVault().accountRecords` for the check.
43
+
44
+ ### Phase 4 — No change needed to `accounts-routes.ts`
45
+
46
+ - `GET /api/vault/accounts` already returns all records including `pending`.
47
+ - `PUT /api/vault/accounts/:id` already handles status updates (pending→accepted/ignored).
48
+
49
+ ## Relevant files
50
+
51
+ - `packages/scraper-app/src/server/check-accounts.ts` — fix `pending` branch
52
+ - `packages/scraper-app/src/server/account-discovery.ts` — new file with
53
+ `registerDiscoveredAccounts`
54
+ - `packages/scraper-app/src/server/websocket.ts` — 2 call sites to add discovery + stale-vault fix
55
+ - `packages/scraper-app/src/server/vault-store.ts` — `updateVault` / `getVault` (read-only
56
+ reference)
57
+ - `packages/scraper-app/src/server/vault.ts` — `AccountRecord` type (read-only reference)
58
+
59
+ ## Verification
60
+
61
+ 1. Run `yarn workspace @accounter-helper/scraper-app test` — existing tests must pass.
62
+ 2. Manual: run a scrape with empty `accountRecords`; confirm new `pending` records appear in
63
+ `GET /api/vault/accounts`.
64
+ 3. Manual: task should be blocked with the pending account IDs shown.
65
+ 4. Manual: classify an account via `PUT /api/vault/accounts/:id` (status=accepted); re-run; confirm
66
+ upload proceeds.
67
+ 5. Manual: re-run with same accounts; confirm no duplicate `accountRecords` are created.
68
+
69
+ ## Decisions
70
+
71
+ - `pending` is treated identically to "not found" from the task's perspective (blocks upload).
72
+ - Discovery runs on the filtered payload (after accepted/ignored filter), so ignored-credential
73
+ accounts are never registered.
74
+ - One `updateVault` call per task (batch all new records), not one per identifier.
75
+ - `branchNumber` is populated for Poalim (from `retrievalTransactionData`); left undefined for card
76
+ sources.
package/index.html ADDED
@@ -0,0 +1,12 @@
1
+ <!doctype html>
2
+ <html lang="en">
3
+ <head>
4
+ <meta charset="UTF-8" />
5
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
6
+ <title>Scraper App</title>
7
+ </head>
8
+ <body>
9
+ <div id="root"></div>
10
+ <script type="module" src="/src/ui/main.tsx"></script>
11
+ </body>
12
+ </html>
package/package.json ADDED
@@ -0,0 +1,40 @@
1
+ {
2
+ "name": "@accounter/scraper-app",
3
+ "version": "0.0.1",
4
+ "type": "module",
5
+ "description": "Scraper app with Fastify server and React UI",
6
+ "license": "MIT",
7
+ "scripts": {
8
+ "build": "yarn build:server && yarn build:ui",
9
+ "build:server": "tsup",
10
+ "build:ui": "vite build",
11
+ "dev:server": "tsup --watch --onSuccess 'node dist/server/index.js'",
12
+ "dev:ui": "vite",
13
+ "test": "vitest run",
14
+ "typecheck": "tsc --noEmit"
15
+ },
16
+ "dependencies": {
17
+ "@accounter/modern-poalim-scraper": "workspace:^",
18
+ "@fastify/static": "9.1.3",
19
+ "@fastify/websocket": "11.2.0",
20
+ "date-fns": "4.1.0",
21
+ "fast-xml-parser": "5.7.2",
22
+ "fastify": "5.8.5",
23
+ "graphql-request": "7.4.0",
24
+ "msw": "2.13.6",
25
+ "react": "19.2.5",
26
+ "react-dom": "19.2.5",
27
+ "ws": "8.20.0",
28
+ "zod": "4.3.6"
29
+ },
30
+ "devDependencies": {
31
+ "@testing-library/dom": "10.4.1",
32
+ "@testing-library/react": "16.3.2",
33
+ "@testing-library/user-event": "14.6.1",
34
+ "@vitejs/plugin-react": "6.0.1",
35
+ "tsup": "8.5.1",
36
+ "typescript": "6.0.3",
37
+ "vite": "8.0.9",
38
+ "vitest": "4.1.5"
39
+ }
40
+ }
@@ -0,0 +1,2 @@
1
+ PORT=4001
2
+ VAULT_PATH=path/to/.vault
@@ -0,0 +1,133 @@
1
+ import { randomBytes } from 'node:crypto';
2
+ import { rm } from 'node:fs/promises';
3
+ import { tmpdir } from 'node:os';
4
+ import { join } from 'node:path';
5
+ import Fastify, { type FastifyInstance } from 'fastify';
6
+ import { afterEach, beforeEach, describe, expect, it } from 'vitest';
7
+ import { defaultVault, saveVaultFile } from '../vault.js';
8
+ import { lockVault } from '../vault-store.js';
9
+ import { registerVaultRoutes } from '../vault-routes.js';
10
+
11
+ const PASSWORD = 'test-password-123';
12
+
13
+ let vaultPath: string;
14
+ let app: FastifyInstance;
15
+
16
+ beforeEach(async () => {
17
+ vaultPath = join(tmpdir(), `vault-test-${randomBytes(4).toString('hex')}.vault`);
18
+ process.env['VAULT_PATH'] = vaultPath;
19
+
20
+ const vault = defaultVault();
21
+ vault.accountRecords.push({
22
+ id: 'acc-1',
23
+ sourceId: 'src-a',
24
+ sourceType: 'poalim',
25
+ accountNumber: '123456',
26
+ branchNumber: '700',
27
+ status: 'pending',
28
+ });
29
+ await saveVaultFile(vaultPath, vault, PASSWORD);
30
+
31
+ app = Fastify();
32
+ await registerVaultRoutes(app);
33
+ await app.ready();
34
+
35
+ await app.inject({
36
+ method: 'POST',
37
+ url: '/api/vault/unlock',
38
+ payload: { password: PASSWORD },
39
+ });
40
+ });
41
+
42
+ afterEach(async () => {
43
+ lockVault();
44
+ await app.close();
45
+ await rm(vaultPath, { force: true });
46
+ delete process.env['VAULT_PATH'];
47
+ });
48
+
49
+ describe('GET /api/vault/accounts', () => {
50
+ it('returns the pre-seeded account', async () => {
51
+ const res = await app.inject({ method: 'GET', url: '/api/vault/accounts' });
52
+ expect(res.statusCode).toBe(200);
53
+ const list = res.json() as Array<{ id: string; status: string }>;
54
+ expect(list).toHaveLength(1);
55
+ expect(list[0].id).toBe('acc-1');
56
+ expect(list[0].status).toBe('pending');
57
+ });
58
+
59
+ it('returns 401 when vault is locked', async () => {
60
+ lockVault();
61
+ const res = await app.inject({ method: 'GET', url: '/api/vault/accounts' });
62
+ expect(res.statusCode).toBe(401);
63
+ });
64
+ });
65
+
66
+ describe('PUT /api/vault/accounts/:id', () => {
67
+ it('sets status to accepted and returns updated list', async () => {
68
+ const res = await app.inject({
69
+ method: 'PUT',
70
+ url: '/api/vault/accounts/acc-1',
71
+ payload: { status: 'accepted' },
72
+ });
73
+ expect(res.statusCode).toBe(200);
74
+ const list = res.json() as Array<{ id: string; status: string }>;
75
+ expect(list[0].status).toBe('accepted');
76
+ });
77
+
78
+ it('sets status to ignored', async () => {
79
+ const res = await app.inject({
80
+ method: 'PUT',
81
+ url: '/api/vault/accounts/acc-1',
82
+ payload: { status: 'ignored' },
83
+ });
84
+ expect(res.statusCode).toBe(200);
85
+ expect((res.json() as Array<{ status: string }>)[0].status).toBe('ignored');
86
+ });
87
+
88
+ it('persists across re-lock/unlock', async () => {
89
+ await app.inject({
90
+ method: 'PUT',
91
+ url: '/api/vault/accounts/acc-1',
92
+ payload: { status: 'accepted' },
93
+ });
94
+
95
+ lockVault();
96
+ await app.inject({
97
+ method: 'POST',
98
+ url: '/api/vault/unlock',
99
+ payload: { password: PASSWORD },
100
+ });
101
+
102
+ const res = await app.inject({ method: 'GET', url: '/api/vault/accounts' });
103
+ expect((res.json() as Array<{ status: string }>)[0].status).toBe('accepted');
104
+ });
105
+
106
+ it('returns 404 for unknown id', async () => {
107
+ const res = await app.inject({
108
+ method: 'PUT',
109
+ url: '/api/vault/accounts/does-not-exist',
110
+ payload: { status: 'accepted' },
111
+ });
112
+ expect(res.statusCode).toBe(404);
113
+ });
114
+
115
+ it('returns 400 for invalid status value', async () => {
116
+ const res = await app.inject({
117
+ method: 'PUT',
118
+ url: '/api/vault/accounts/acc-1',
119
+ payload: { status: 'pending' },
120
+ });
121
+ expect(res.statusCode).toBe(400);
122
+ });
123
+
124
+ it('returns 401 when vault is locked', async () => {
125
+ lockVault();
126
+ const res = await app.inject({
127
+ method: 'PUT',
128
+ url: '/api/vault/accounts/acc-1',
129
+ payload: { status: 'accepted' },
130
+ });
131
+ expect(res.statusCode).toBe(401);
132
+ });
133
+ });
@@ -0,0 +1,305 @@
1
+ import { afterEach, describe, expect, it } from 'vitest';
2
+ import { checkAccounts } from '../check-accounts.js';
3
+ import type { AccountRecord } from '../check-accounts.js';
4
+ import { _resetRunState, startRun, type ScrapeTask } from '../scrape-runner.js';
5
+ import type { ServerMessage } from '../../shared/ws-protocol.js';
6
+
7
+ const poalimPayload = {
8
+ transactions: [],
9
+ retrievalTransactionData: { accountNumber: 100000, branchNumber: 600, bankNumber: 12 },
10
+ };
11
+
12
+ const isracardPayload = {
13
+ Header: { Status: '1', Message: null },
14
+ CardsTransactionsListBean: {
15
+ cardNumberList: ['CARD 9876', 'CARD 4545'],
16
+ Index0: {
17
+ '@AllCards': 'AllCards',
18
+ CurrentCardTransactions: [
19
+ { txnIsrael: null, txnAbroad: null },
20
+ { txnIsrael: null, txnAbroad: null },
21
+ ],
22
+ },
23
+ },
24
+ };
25
+
26
+ const calPayload = [
27
+ { card: 'ACC-1', month: '2024-01', transactions: [] },
28
+ { card: 'ACC-2', month: '2024-01', transactions: [] },
29
+ ];
30
+
31
+ const maxPayload = [
32
+ { accountNumber: '7', txns: [] },
33
+ { accountNumber: '8', txns: [] },
34
+ ];
35
+
36
+ const discountPayload = [
37
+ { accountNumber: 'ACC-001', month: '2024-01', balance: 5000, transactions: [] },
38
+ { accountNumber: 'ACC-002', month: '2024-01', balance: 3000, transactions: [] },
39
+ ];
40
+
41
+ function makeRecord(
42
+ sourceType: AccountRecord['sourceType'],
43
+ accountNumber: string,
44
+ status: AccountRecord['status'] = 'accepted',
45
+ ): AccountRecord {
46
+ return {
47
+ id: `${sourceType}-${accountNumber}`,
48
+ sourceId: 'src-1',
49
+ sourceType,
50
+ accountNumber,
51
+ status,
52
+ };
53
+ }
54
+
55
+ describe('checkAccounts — poalim', () => {
56
+ it('accepted when account is in known list with accepted status', () => {
57
+ const known = [makeRecord('poalim', '100000', 'accepted')];
58
+ const result = checkAccounts('poalim', poalimPayload, known);
59
+ expect(result.accepted).toEqual(['100000']);
60
+ expect(result.ignored).toEqual([]);
61
+ expect(result.unknown).toEqual([]);
62
+ });
63
+
64
+ it('unknown when account has pending status', () => {
65
+ const known = [makeRecord('poalim', '100000', 'pending')];
66
+ const result = checkAccounts('poalim', poalimPayload, known);
67
+ expect(result.accepted).toEqual([]);
68
+ expect(result.unknown).toEqual(['100000']);
69
+ });
70
+
71
+ it('unknown when account is not in known list', () => {
72
+ const result = checkAccounts('poalim', poalimPayload, []);
73
+ expect(result.unknown).toEqual(['100000']);
74
+ expect(result.accepted).toEqual([]);
75
+ });
76
+
77
+ it('ignored when account status is ignored', () => {
78
+ const known = [makeRecord('poalim', '100000', 'ignored')];
79
+ const result = checkAccounts('poalim', poalimPayload, known);
80
+ expect(result.ignored).toEqual(['100000']);
81
+ expect(result.accepted).toEqual([]);
82
+ expect(result.unknown).toEqual([]);
83
+ });
84
+
85
+ it('does not match account from a different sourceType', () => {
86
+ const known = [makeRecord('discount', '100000', 'accepted')];
87
+ const result = checkAccounts('poalim', poalimPayload, known);
88
+ expect(result.unknown).toEqual(['100000']);
89
+ });
90
+ });
91
+
92
+ describe('checkAccounts — discount', () => {
93
+ it('classifies accountNumber identifiers from payload', () => {
94
+ const known = [
95
+ makeRecord('discount', 'ACC-001', 'accepted'),
96
+ makeRecord('discount', 'ACC-002', 'ignored'),
97
+ ];
98
+ const result = checkAccounts('discount', discountPayload, known);
99
+ expect(result.accepted).toEqual(['ACC-001']);
100
+ expect(result.ignored).toEqual(['ACC-002']);
101
+ expect(result.unknown).toEqual([]);
102
+ });
103
+
104
+ it('returns unknown for unrecognised accountNumber', () => {
105
+ const result = checkAccounts('discount', discountPayload, []);
106
+ expect(result.unknown).toEqual(['ACC-001', 'ACC-002']);
107
+ expect(result.accepted).toEqual([]);
108
+ });
109
+
110
+ it('deduplicates accountNumber across multiple months for the same account', () => {
111
+ const sameAccountTwoMonths = [
112
+ { accountNumber: 'ACC-001', month: '2024-01', balance: 5000, transactions: [] },
113
+ { accountNumber: 'ACC-001', month: '2024-02', balance: 4800, transactions: [] },
114
+ ];
115
+ const result = checkAccounts('discount', sameAccountTwoMonths, []);
116
+ expect(result.unknown).toEqual(['ACC-001']);
117
+ });
118
+ });
119
+
120
+ describe('checkAccounts — isracard / amex', () => {
121
+ it('classifies each card from CurrentCardTransactions', () => {
122
+ const known = [
123
+ makeRecord('isracard', '4545', 'accepted'),
124
+ makeRecord('isracard', '9876', 'ignored'),
125
+ ];
126
+ const result = checkAccounts('isracard', isracardPayload, known);
127
+ expect(result.accepted).toEqual(['4545']);
128
+ expect(result.ignored).toEqual(['9876']);
129
+ expect(result.unknown).toEqual([]);
130
+ });
131
+
132
+ it('returns unknown cards not in known list', () => {
133
+ const known = [makeRecord('isracard', '9876', 'accepted')];
134
+ const result = checkAccounts('isracard', isracardPayload, known);
135
+ expect(result.unknown).toEqual(['4545']);
136
+ });
137
+
138
+ it('works the same for amex payload type', () => {
139
+ const known = [makeRecord('amex', '4545', 'ignored')];
140
+ const result = checkAccounts('amex', isracardPayload, known);
141
+ expect(result.ignored).toEqual(['4545']);
142
+ expect(result.unknown).toEqual(['9876']);
143
+ });
144
+
145
+ it('returns empty when there are no cards in payload', () => {
146
+ const emptyPayload = {
147
+ Header: { Status: '1', Message: null },
148
+ CardsTransactionsListBean: {
149
+ cardNumberList: [],
150
+ Index0: { '@AllCards': 'AllCards', CurrentCardTransactions: [] },
151
+ },
152
+ };
153
+ const result = checkAccounts('isracard', emptyPayload, []);
154
+ expect(result).toEqual({ accepted: [], ignored: [], unknown: [] });
155
+ });
156
+
157
+ it('collects identifiers from cardNumberList', () => {
158
+ const multiIndexPayload = {
159
+ Header: { Status: '1', Message: null },
160
+ CardsTransactionsListBean: {
161
+ cardNumberList: ['CARD 1111', 'CARD 2222', 'CARD 3333'],
162
+ Index0: {
163
+ '@AllCards': 'AllCards',
164
+ CurrentCardTransactions: [{ '@cardTransactions': 'CARD-A' }],
165
+ },
166
+ Index1: {
167
+ '@AllCards': 'AllCards',
168
+ CurrentCardTransactions: [{ '@cardTransactions': 'CARD-B' }],
169
+ },
170
+ Index2: {
171
+ '@AllCards': 'AllCards',
172
+ CurrentCardTransactions: [{ '@cardTransactions': 'CARD-C' }],
173
+ },
174
+ },
175
+ };
176
+ const result = checkAccounts('isracard', multiIndexPayload, []);
177
+ expect(result.unknown.sort()).toEqual(['1111', '2222', '3333']);
178
+ });
179
+
180
+ it('ignores non-Index* keys in CardsTransactionsListBean', () => {
181
+ const payloadWithExtraKeys = {
182
+ Header: { Status: '1', Message: null },
183
+ CardsTransactionsListBean: {
184
+ cardNumberList: ['CARD 0101'],
185
+ Index0: {
186
+ '@AllCards': 'AllCards',
187
+ CurrentCardTransactions: [],
188
+ },
189
+ cardIdx: '0',
190
+ card0: { some: 'data' },
191
+ },
192
+ };
193
+ const result = checkAccounts('isracard', payloadWithExtraKeys, []);
194
+ expect(result.unknown).toEqual(['0101']);
195
+ });
196
+ });
197
+
198
+ describe('checkAccounts — cal', () => {
199
+ it('classifies card identifiers', () => {
200
+ const known = [
201
+ makeRecord('cal', 'ACC-1', 'accepted'),
202
+ makeRecord('cal', 'ACC-2', 'ignored'),
203
+ ];
204
+ const result = checkAccounts('cal', calPayload, known);
205
+ expect(result.accepted).toEqual(['ACC-1']);
206
+ expect(result.ignored).toEqual(['ACC-2']);
207
+ expect(result.unknown).toEqual([]);
208
+ });
209
+
210
+ it('returns unknown for unrecognised card', () => {
211
+ const result = checkAccounts('cal', calPayload, []);
212
+ expect(result.unknown).toEqual(['ACC-1', 'ACC-2']);
213
+ });
214
+ });
215
+
216
+ describe('checkAccounts — max', () => {
217
+ it('classifies unique cardIndex values from transactions', () => {
218
+ const known = [
219
+ makeRecord('max', '7', 'accepted'),
220
+ makeRecord('max', '8', 'ignored'),
221
+ ];
222
+ const result = checkAccounts('max', maxPayload, known);
223
+ expect(result.accepted).toEqual(['7']);
224
+ expect(result.ignored).toEqual(['8']);
225
+ expect(result.unknown).toEqual([]);
226
+ });
227
+
228
+ it('deduplicates accountNumber across multiple entries for the same card', () => {
229
+ const twoEntriesSameCard = [
230
+ { accountNumber: '7', txns: [] },
231
+ { accountNumber: '7', txns: [] },
232
+ ];
233
+ const result = checkAccounts('max', twoEntriesSameCard, []);
234
+ expect(result.unknown).toEqual(['7']);
235
+ });
236
+
237
+ it('returns empty arrays when there are no accounts', () => {
238
+ const result = checkAccounts('max', [], []);
239
+ expect(result).toEqual({ accepted: [], ignored: [], unknown: [] });
240
+ });
241
+ });
242
+
243
+ describe('runner integration — task-blocked on unknown accounts', () => {
244
+ afterEach(() => {
245
+ _resetRunState();
246
+ });
247
+
248
+ it('emits task-blocked (not a crash) when checkAccounts finds unknown accounts', async () => {
249
+ const events: ServerMessage[] = [];
250
+ const emit = (msg: ServerMessage) => events.push(msg);
251
+
252
+ const task: ScrapeTask = {
253
+ sourceId: 'poalim-src',
254
+ nickname: 'poalim-src',
255
+ type: 'poalim',
256
+ run: async () => {
257
+ const check = checkAccounts('poalim', poalimPayload, []);
258
+ if (check.unknown.length > 0) {
259
+ emit({ type: 'task-blocked', sourceId: 'poalim-src', sourceType: 'poalim', unknownAccounts: check.unknown });
260
+ return { inserted: 0, skipped: 0, insertedIds: [] };
261
+ }
262
+ return { inserted: 1, skipped: 0, insertedIds: ['x'] };
263
+ },
264
+ };
265
+
266
+ await startRun([task], false, emit);
267
+
268
+ const blocked = events.find(e => e.type === 'task-blocked');
269
+ expect(blocked).toBeTruthy();
270
+ expect((blocked as { unknownAccounts: string[] }).unknownAccounts).toContain('100000');
271
+ expect(events.at(-1)).toMatchObject({ type: 'run-complete', totalInserted: 0, totalSkipped: 0 });
272
+ });
273
+
274
+ it('continues remaining tasks after a blocked task', async () => {
275
+ const events: ServerMessage[] = [];
276
+ const emit = (msg: ServerMessage) => events.push(msg);
277
+
278
+ const blockedTask: ScrapeTask = {
279
+ sourceId: 'src-1',
280
+ nickname: 'src-1',
281
+ type: 'poalim',
282
+ run: async () => {
283
+ const check = checkAccounts('poalim', poalimPayload, []);
284
+ if (check.unknown.length > 0) {
285
+ emit({ type: 'task-blocked', sourceId: 'src-1', sourceType: 'poalim', unknownAccounts: check.unknown });
286
+ return { inserted: 0, skipped: 0, insertedIds: [] };
287
+ }
288
+ return { inserted: 1, skipped: 0, insertedIds: ['x'] };
289
+ },
290
+ };
291
+
292
+ const normalTask: ScrapeTask = {
293
+ sourceId: 'src-2',
294
+ nickname: 'src-2',
295
+ type: 'poalim',
296
+ run: async () => ({ inserted: 2, skipped: 1, insertedIds: ['a', 'b'] }),
297
+ };
298
+
299
+ await startRun([blockedTask, normalTask], false, emit);
300
+
301
+ expect(events.some(e => e.type === 'task-blocked')).toBe(true);
302
+ expect(events.some(e => e.type === 'task-done' && 'sourceId' in e && e.sourceId === 'src-2')).toBe(true);
303
+ expect(events.at(-1)).toMatchObject({ type: 'run-complete', totalInserted: 2, totalSkipped: 1 });
304
+ });
305
+ });