@accounter/scraper-app 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +90 -0
- package/docs/plan.md +76 -0
- package/index.html +12 -0
- package/package.json +40 -0
- package/src/env.template +2 -0
- package/src/server/__tests__/accounts-routes.test.ts +133 -0
- package/src/server/__tests__/check-accounts.test.ts +305 -0
- package/src/server/__tests__/filter-payload.test.ts +193 -0
- package/src/server/__tests__/graphql-client.integration.test.ts +98 -0
- package/src/server/__tests__/graphql-client.test.ts +508 -0
- package/src/server/__tests__/healthz.test.ts +22 -0
- package/src/server/__tests__/history.test.ts +111 -0
- package/src/server/__tests__/otp-manager.test.ts +132 -0
- package/src/server/__tests__/scrape-runner.test.ts +144 -0
- package/src/server/__tests__/settings-routes.test.ts +117 -0
- package/src/server/__tests__/sources-routes.test.ts +149 -0
- package/src/server/__tests__/validate-payload.test.ts +193 -0
- package/src/server/__tests__/vault-routes.test.ts +174 -0
- package/src/server/__tests__/vault.test.ts +33 -0
- package/src/server/__tests__/websocket.test.ts +151 -0
- package/src/server/account-discovery.ts +49 -0
- package/src/server/accounts-routes.ts +74 -0
- package/src/server/check-accounts.ts +79 -0
- package/src/server/filter-payload.ts +145 -0
- package/src/server/graphql/client.ts +103 -0
- package/src/server/graphql/mutations.ts +518 -0
- package/src/server/history-routes.ts +11 -0
- package/src/server/history.ts +53 -0
- package/src/server/index.ts +40 -0
- package/src/server/otp-manager.ts +63 -0
- package/src/server/payload-schemas/amex.schema.ts +2 -0
- package/src/server/payload-schemas/cal.schema.ts +27 -0
- package/src/server/payload-schemas/currency-rates.schema.ts +11 -0
- package/src/server/payload-schemas/discount.schema.ts +26 -0
- package/src/server/payload-schemas/isracard.schema.ts +58 -0
- package/src/server/payload-schemas/max.schema.ts +27 -0
- package/src/server/payload-schemas/poalim-foreign.schema.ts +30 -0
- package/src/server/payload-schemas/poalim-ils.schema.ts +31 -0
- package/src/server/payload-schemas/poalim-swift.schema.ts +21 -0
- package/src/server/scrape-runner.ts +165 -0
- package/src/server/scrapers/__tests__/amex.test.ts +142 -0
- package/src/server/scrapers/__tests__/cal.test.ts +135 -0
- package/src/server/scrapers/__tests__/currency-rates.test.ts +105 -0
- package/src/server/scrapers/__tests__/discount.test.ts +160 -0
- package/src/server/scrapers/__tests__/isracard.test.ts +142 -0
- package/src/server/scrapers/__tests__/max.test.ts +115 -0
- package/src/server/scrapers/__tests__/poalim.test.ts +154 -0
- package/src/server/scrapers/amex.ts +63 -0
- package/src/server/scrapers/cal.ts +56 -0
- package/src/server/scrapers/currency-rates.ts +64 -0
- package/src/server/scrapers/discount.ts +62 -0
- package/src/server/scrapers/isracard.ts +68 -0
- package/src/server/scrapers/max.ts +32 -0
- package/src/server/scrapers/poalim.ts +103 -0
- package/src/server/settings-routes.ts +27 -0
- package/src/server/sources-routes.ts +182 -0
- package/src/server/validate-payload.ts +74 -0
- package/src/server/vault-routes.ts +99 -0
- package/src/server/vault-store.ts +42 -0
- package/src/server/vault.ts +216 -0
- package/src/server/websocket.ts +454 -0
- package/src/shared/source-types.ts +10 -0
- package/src/shared/types.ts +20 -0
- package/src/shared/ws-protocol.ts +177 -0
- package/src/test-setup.ts +6 -0
- package/src/ui/__tests__/accounts-tab.test.tsx +134 -0
- package/src/ui/__tests__/config.test.tsx +99 -0
- package/src/ui/__tests__/history.test.tsx +94 -0
- package/src/ui/__tests__/run.test.tsx +195 -0
- package/src/ui/__tests__/settings-tab.test.tsx +79 -0
- package/src/ui/__tests__/sources-tab.test.tsx +139 -0
- package/src/ui/__tests__/vault-setup.test.tsx +105 -0
- package/src/ui/__tests__/vault-unlock.test.tsx +78 -0
- package/src/ui/app.tsx +109 -0
- package/src/ui/components/error-boundary.tsx +54 -0
- package/src/ui/components/otp-modal.tsx +82 -0
- package/src/ui/components/skeleton.tsx +58 -0
- package/src/ui/components/task-row.tsx +241 -0
- package/src/ui/contexts/vault-context.tsx +77 -0
- package/src/ui/lib/api.ts +117 -0
- package/src/ui/lib/ws.ts +137 -0
- package/src/ui/main.tsx +9 -0
- package/src/ui/screens/config/accounts-tab.tsx +185 -0
- package/src/ui/screens/config/config.tsx +163 -0
- package/src/ui/screens/config/settings-tab.tsx +167 -0
- package/src/ui/screens/config/source-forms.tsx +518 -0
- package/src/ui/screens/config/source-types.ts +91 -0
- package/src/ui/screens/config/sources-tab.tsx +176 -0
- package/src/ui/screens/history.tsx +234 -0
- package/src/ui/screens/run.tsx +266 -0
- package/src/ui/screens/vault-setup.tsx +120 -0
- package/src/ui/screens/vault-unlock.tsx +38 -0
- package/tsconfig.json +15 -0
- package/tsup.config.ts +10 -0
- package/vite.config.ts +24 -0
- package/vitest.config.ts +7 -0
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { addMonths, startOfMonth } from 'date-fns';
|
|
2
|
+
import type { z } from 'zod';
|
|
3
|
+
import { init, type IsracardCardsTransactionsList } from '@accounter/modern-poalim-scraper';
|
|
4
|
+
import type { ServerMessage } from '../../shared/ws-protocol.js';
|
|
5
|
+
import { validatePayload } from '../validate-payload.js';
|
|
6
|
+
import type { IsracardAmexAccountSchema } from '../vault.js';
|
|
7
|
+
|
|
8
|
+
export type AmexCreds = z.infer<typeof IsracardAmexAccountSchema>;
|
|
9
|
+
|
|
10
|
+
export type Emitter = (msg: ServerMessage) => void;
|
|
11
|
+
|
|
12
|
+
function buildMonthList(dateFrom: Date, dateTo: Date): Date[] {
|
|
13
|
+
const months: Date[] = [];
|
|
14
|
+
let current = startOfMonth(dateFrom);
|
|
15
|
+
const end = startOfMonth(dateTo);
|
|
16
|
+
while (current <= end) {
|
|
17
|
+
months.push(current);
|
|
18
|
+
current = addMonths(current, 1);
|
|
19
|
+
}
|
|
20
|
+
return months;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export async function scrapeAmex(
|
|
24
|
+
creds: AmexCreds,
|
|
25
|
+
dateFrom: Date,
|
|
26
|
+
dateTo: Date,
|
|
27
|
+
emit: Emitter,
|
|
28
|
+
): Promise<IsracardCardsTransactionsList[]> {
|
|
29
|
+
const { amex: amexFn, close } = await init({ headless: true });
|
|
30
|
+
|
|
31
|
+
try {
|
|
32
|
+
const scraper = await amexFn(
|
|
33
|
+
{ ID: creds.ownerId, password: creds.password, card6Digits: creds.last6Digits },
|
|
34
|
+
{ validateSchema: true },
|
|
35
|
+
);
|
|
36
|
+
|
|
37
|
+
const months = buildMonthList(dateFrom, dateTo);
|
|
38
|
+
const results: IsracardCardsTransactionsList[] = [];
|
|
39
|
+
|
|
40
|
+
for (const month of months) {
|
|
41
|
+
emit({ type: 'scrape-progress', sourceId: creds.id, sourceType: 'amex', status: 'running' });
|
|
42
|
+
const { data, isValid } = await scraper.getMonthTransactions(month);
|
|
43
|
+
|
|
44
|
+
if (!data) continue;
|
|
45
|
+
|
|
46
|
+
if (!isValid) {
|
|
47
|
+
throw new Error(`Invalid Amex data for ${month.toISOString().slice(0, 7)}`);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
if (data.Header?.Status !== '1') {
|
|
51
|
+
throw new Error(
|
|
52
|
+
`Amex login/password issue (Header.Status=${data.Header?.Status}) for ${month.toISOString().slice(0, 7)}`,
|
|
53
|
+
);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
results.push(validatePayload('amex', data));
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
return results;
|
|
60
|
+
} finally {
|
|
61
|
+
await close();
|
|
62
|
+
}
|
|
63
|
+
}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import { addMonths, format, startOfMonth } from 'date-fns';
|
|
2
|
+
import type { z } from 'zod';
|
|
3
|
+
import { init } from '@accounter/modern-poalim-scraper';
|
|
4
|
+
import type { ServerMessage } from '../../shared/ws-protocol.js';
|
|
5
|
+
import type { CalPayload } from '../payload-schemas/cal.schema.js';
|
|
6
|
+
import { validatePayload } from '../validate-payload.js';
|
|
7
|
+
import type { CalAccountSchema } from '../vault.js';
|
|
8
|
+
|
|
9
|
+
export type CalCreds = z.infer<typeof CalAccountSchema>;
|
|
10
|
+
|
|
11
|
+
export type Emitter = (msg: ServerMessage) => void;
|
|
12
|
+
|
|
13
|
+
function buildMonthList(dateFrom: Date, dateTo: Date): Date[] {
|
|
14
|
+
const months: Date[] = [];
|
|
15
|
+
let current = startOfMonth(dateFrom);
|
|
16
|
+
const end = startOfMonth(dateTo);
|
|
17
|
+
while (current <= end) {
|
|
18
|
+
months.push(current);
|
|
19
|
+
current = addMonths(current, 1);
|
|
20
|
+
}
|
|
21
|
+
return months;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export async function scrapeCal(
|
|
25
|
+
creds: CalCreds,
|
|
26
|
+
dateFrom: Date,
|
|
27
|
+
dateTo: Date,
|
|
28
|
+
emit: Emitter,
|
|
29
|
+
): Promise<CalPayload> {
|
|
30
|
+
const { cal: calFn, close } = await init({ headless: true });
|
|
31
|
+
|
|
32
|
+
try {
|
|
33
|
+
const scraper = await calFn({
|
|
34
|
+
username: creds.username,
|
|
35
|
+
password: creds.password,
|
|
36
|
+
last4Digits: creds.last4Digits,
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
const months = buildMonthList(dateFrom, dateTo);
|
|
40
|
+
const results: CalPayload = [];
|
|
41
|
+
|
|
42
|
+
for (const month of months) {
|
|
43
|
+
emit({ type: 'scrape-progress', sourceId: creds.id, sourceType: 'cal', status: 'running' });
|
|
44
|
+
const transactions = await scraper.getMonthTransactions(creds.last4Digits, month);
|
|
45
|
+
results.push({
|
|
46
|
+
card: creds.last4Digits,
|
|
47
|
+
month: format(month, 'yyyy-MM'),
|
|
48
|
+
transactions,
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
return validatePayload('cal', results);
|
|
53
|
+
} finally {
|
|
54
|
+
await close();
|
|
55
|
+
}
|
|
56
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { format } from 'date-fns';
|
|
2
|
+
import { XMLParser } from 'fast-xml-parser';
|
|
3
|
+
import type { ServerMessage } from '../../shared/ws-protocol.js';
|
|
4
|
+
import type { CurrencyRatesPayload } from '../payload-schemas/currency-rates.schema.js';
|
|
5
|
+
import { validatePayload } from '../validate-payload.js';
|
|
6
|
+
|
|
7
|
+
const BOI_URL =
|
|
8
|
+
'https://edge.boi.gov.il/FusionEdgeServer/sdmx/v2/data/dataflow/BOI.STATISTICS/EXR/1.0/RER_USD_ILS,RER_EUR_ILS,RER_GBP_ILS,RER_CAD_ILS,RER_JPY_ILS,RER_AUD_ILS,RER_SEK_ILS';
|
|
9
|
+
|
|
10
|
+
const CURRENCIES = ['USD', 'EUR', 'GBP', 'CAD', 'JPY', 'AUD', 'SEK'] as const;
|
|
11
|
+
type SupportedCurrency = (typeof CURRENCIES)[number];
|
|
12
|
+
|
|
13
|
+
export type Emitter = (msg: ServerMessage) => void;
|
|
14
|
+
|
|
15
|
+
export async function scrapeCurrencyRates(_emit: Emitter): Promise<CurrencyRatesPayload> {
|
|
16
|
+
const res = await fetch(BOI_URL);
|
|
17
|
+
if (!res.ok) throw new Error(`BOI fetch failed: ${res.status} ${res.statusText}`);
|
|
18
|
+
|
|
19
|
+
const xml = await res.text();
|
|
20
|
+
const parser = new XMLParser({
|
|
21
|
+
ignoreAttributes: false,
|
|
22
|
+
isArray: name => name === 'Series' || name === 'Obs',
|
|
23
|
+
removeNSPrefix: true,
|
|
24
|
+
});
|
|
25
|
+
const parsed = parser.parse(xml);
|
|
26
|
+
const series: unknown[] = parsed?.StructureSpecificData?.DataSet?.Series ?? [];
|
|
27
|
+
|
|
28
|
+
if (series.length === 0) {
|
|
29
|
+
return validatePayload('currency-rates', []);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const today = format(new Date(), 'yyyy-MM-dd');
|
|
33
|
+
|
|
34
|
+
// Collect per-date, per-currency rates
|
|
35
|
+
const rateMap = new Map<string, Map<SupportedCurrency, number>>();
|
|
36
|
+
|
|
37
|
+
for (const currencyData of series) {
|
|
38
|
+
const cd = currencyData as Record<string, unknown>;
|
|
39
|
+
const currency = cd['@_BASE_CURRENCY'] as SupportedCurrency;
|
|
40
|
+
if (!CURRENCIES.includes(currency)) continue;
|
|
41
|
+
|
|
42
|
+
const obs: unknown[] = Array.isArray(cd['Obs']) ? (cd['Obs'] as unknown[]) : [];
|
|
43
|
+
for (const entry of obs) {
|
|
44
|
+
const e = entry as Record<string, string>;
|
|
45
|
+
const date = e['@_TIME_PERIOD'];
|
|
46
|
+
const value = Number(e['@_OBS_VALUE']);
|
|
47
|
+
if (!date || Number.isNaN(value) || date === today) continue;
|
|
48
|
+
|
|
49
|
+
const rate = currency === 'JPY' ? value * 0.01 : value;
|
|
50
|
+
|
|
51
|
+
if (!rateMap.has(date)) rateMap.set(date, new Map());
|
|
52
|
+
rateMap.get(date)!.set(currency, rate);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
const entries: { date: string; currency: SupportedCurrency; rate: number }[] = [];
|
|
57
|
+
for (const [date, currencyRates] of rateMap) {
|
|
58
|
+
for (const [currency, rate] of currencyRates) {
|
|
59
|
+
entries.push({ date, currency, rate });
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
return validatePayload('currency-rates', entries);
|
|
64
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { addMonths, format, startOfMonth } from 'date-fns';
|
|
2
|
+
import type { z } from 'zod';
|
|
3
|
+
import { init } from '@accounter/modern-poalim-scraper';
|
|
4
|
+
import type { ServerMessage } from '../../shared/ws-protocol.js';
|
|
5
|
+
import type { DiscountPayload } from '../payload-schemas/discount.schema.js';
|
|
6
|
+
import { validatePayload } from '../validate-payload.js';
|
|
7
|
+
import type { DiscountAccountSchema } from '../vault.js';
|
|
8
|
+
|
|
9
|
+
export type DiscountCreds = z.infer<typeof DiscountAccountSchema>;
|
|
10
|
+
|
|
11
|
+
export type Emitter = (msg: ServerMessage) => void;
|
|
12
|
+
|
|
13
|
+
function buildMonthList(dateFrom: Date, dateTo: Date): Date[] {
|
|
14
|
+
const months: Date[] = [];
|
|
15
|
+
let current = startOfMonth(dateFrom);
|
|
16
|
+
const end = startOfMonth(dateTo);
|
|
17
|
+
while (current <= end) {
|
|
18
|
+
months.push(current);
|
|
19
|
+
current = addMonths(current, 1);
|
|
20
|
+
}
|
|
21
|
+
return months;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export async function scrapeDiscount(
|
|
25
|
+
creds: DiscountCreds,
|
|
26
|
+
dateFrom: Date,
|
|
27
|
+
dateTo: Date,
|
|
28
|
+
emit: Emitter,
|
|
29
|
+
): Promise<DiscountPayload> {
|
|
30
|
+
const { discount: discountFn, close } = await init({ headless: true });
|
|
31
|
+
|
|
32
|
+
try {
|
|
33
|
+
const scraper = await discountFn({
|
|
34
|
+
ID: creds.ID,
|
|
35
|
+
password: creds.password,
|
|
36
|
+
...(creds.code ? { code: creds.code } : {}),
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
const months = buildMonthList(dateFrom, dateTo);
|
|
40
|
+
const results: DiscountPayload = [];
|
|
41
|
+
|
|
42
|
+
for (const month of months) {
|
|
43
|
+
emit({
|
|
44
|
+
type: 'scrape-progress',
|
|
45
|
+
sourceId: creds.id,
|
|
46
|
+
sourceType: 'discount',
|
|
47
|
+
status: 'running',
|
|
48
|
+
});
|
|
49
|
+
const { accountNumber, balance, transactions } = await scraper.getMonthTransactions(month);
|
|
50
|
+
results.push({
|
|
51
|
+
accountNumber,
|
|
52
|
+
month: format(month, 'yyyy-MM'),
|
|
53
|
+
balance,
|
|
54
|
+
transactions,
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
return validatePayload('discount', results);
|
|
59
|
+
} finally {
|
|
60
|
+
await close();
|
|
61
|
+
}
|
|
62
|
+
}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import { addMonths, startOfMonth } from 'date-fns';
|
|
2
|
+
import type { z } from 'zod';
|
|
3
|
+
import { init, type IsracardCardsTransactionsList } from '@accounter/modern-poalim-scraper';
|
|
4
|
+
import type { ServerMessage } from '../../shared/ws-protocol.js';
|
|
5
|
+
import { validatePayload } from '../validate-payload.js';
|
|
6
|
+
import type { IsracardAmexAccountSchema } from '../vault.js';
|
|
7
|
+
|
|
8
|
+
export type IsracardCreds = z.infer<typeof IsracardAmexAccountSchema>;
|
|
9
|
+
|
|
10
|
+
export type Emitter = (msg: ServerMessage) => void;
|
|
11
|
+
|
|
12
|
+
function buildMonthList(dateFrom: Date, dateTo: Date): Date[] {
|
|
13
|
+
const months: Date[] = [];
|
|
14
|
+
let current = startOfMonth(dateFrom);
|
|
15
|
+
const end = startOfMonth(dateTo);
|
|
16
|
+
while (current <= end) {
|
|
17
|
+
months.push(current);
|
|
18
|
+
current = addMonths(current, 1);
|
|
19
|
+
}
|
|
20
|
+
return months;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export async function scrapeIsracard(
|
|
24
|
+
creds: IsracardCreds,
|
|
25
|
+
dateFrom: Date,
|
|
26
|
+
dateTo: Date,
|
|
27
|
+
emit: Emitter,
|
|
28
|
+
): Promise<IsracardCardsTransactionsList[]> {
|
|
29
|
+
const { isracard: isracardFn, close } = await init({ headless: true });
|
|
30
|
+
|
|
31
|
+
try {
|
|
32
|
+
const scraper = await isracardFn(
|
|
33
|
+
{ ID: creds.ownerId, password: creds.password, card6Digits: creds.last6Digits },
|
|
34
|
+
{ validateSchema: true },
|
|
35
|
+
);
|
|
36
|
+
|
|
37
|
+
const months = buildMonthList(dateFrom, dateTo);
|
|
38
|
+
const results: IsracardCardsTransactionsList[] = [];
|
|
39
|
+
|
|
40
|
+
for (const month of months) {
|
|
41
|
+
emit({
|
|
42
|
+
type: 'scrape-progress',
|
|
43
|
+
sourceId: creds.id,
|
|
44
|
+
sourceType: 'isracard',
|
|
45
|
+
status: 'running',
|
|
46
|
+
});
|
|
47
|
+
const { data, isValid } = await scraper.getMonthTransactions(month);
|
|
48
|
+
|
|
49
|
+
if (!data) continue;
|
|
50
|
+
|
|
51
|
+
if (!isValid) {
|
|
52
|
+
throw new Error(`Invalid Isracard data for ${month.toISOString().slice(0, 7)}`);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
if (data.Header?.Status !== '1') {
|
|
56
|
+
throw new Error(
|
|
57
|
+
`Isracard login/password issue (Header.Status=${data.Header?.Status}) for ${month.toISOString().slice(0, 7)}`,
|
|
58
|
+
);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
results.push(validatePayload('isracard', data));
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
return results;
|
|
65
|
+
} finally {
|
|
66
|
+
await close();
|
|
67
|
+
}
|
|
68
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import type { z } from 'zod';
|
|
2
|
+
import { init } from '@accounter/modern-poalim-scraper';
|
|
3
|
+
import type { ServerMessage } from '../../shared/ws-protocol.js';
|
|
4
|
+
import type { MaxPayload } from '../payload-schemas/max.schema.js';
|
|
5
|
+
import { validatePayload } from '../validate-payload.js';
|
|
6
|
+
import type { MaxAccountSchema } from '../vault.js';
|
|
7
|
+
|
|
8
|
+
export type MaxCreds = z.infer<typeof MaxAccountSchema>;
|
|
9
|
+
|
|
10
|
+
export type Emitter = (msg: ServerMessage) => void;
|
|
11
|
+
|
|
12
|
+
export async function scrapeMax(
|
|
13
|
+
creds: MaxCreds,
|
|
14
|
+
dateFrom: Date,
|
|
15
|
+
_dateTo: Date,
|
|
16
|
+
emit: Emitter,
|
|
17
|
+
): Promise<MaxPayload> {
|
|
18
|
+
const { max: maxFn, close } = await init({ headless: true });
|
|
19
|
+
|
|
20
|
+
try {
|
|
21
|
+
const scraper = await maxFn(
|
|
22
|
+
{ username: creds.username, password: creds.password },
|
|
23
|
+
{ startDate: dateFrom },
|
|
24
|
+
);
|
|
25
|
+
|
|
26
|
+
emit({ type: 'scrape-progress', sourceId: creds.id, sourceType: 'max', status: 'running' });
|
|
27
|
+
const accounts = await scraper.getTransactions();
|
|
28
|
+
return validatePayload('max', accounts);
|
|
29
|
+
} finally {
|
|
30
|
+
await close();
|
|
31
|
+
}
|
|
32
|
+
}
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
import type { z } from 'zod';
|
|
2
|
+
import { init } from '@accounter/modern-poalim-scraper';
|
|
3
|
+
import type { ServerMessage } from '../../shared/ws-protocol.js';
|
|
4
|
+
import type { OtpManager } from '../otp-manager.js';
|
|
5
|
+
import type { PoalimForeignPayload } from '../payload-schemas/poalim-foreign.schema.js';
|
|
6
|
+
import type { PoalimIlsPayload } from '../payload-schemas/poalim-ils.schema.js';
|
|
7
|
+
import type { PoalimSwiftPayload } from '../payload-schemas/poalim-swift.schema.js';
|
|
8
|
+
import { validatePayload } from '../validate-payload.js';
|
|
9
|
+
import type { PoalimAccountSchema } from '../vault.js';
|
|
10
|
+
|
|
11
|
+
export type PoalimCreds = z.infer<typeof PoalimAccountSchema>;
|
|
12
|
+
|
|
13
|
+
const OTP_TIMEOUT_MS = 5 * 60 * 1000;
|
|
14
|
+
|
|
15
|
+
// Local options type that extends the scraper's options with an OTP callback.
|
|
16
|
+
// The hapoalim scraper uses inquirer for OTP today; otpCallback is the hook
|
|
17
|
+
// for WS-based OTP flow. The cast below is required until the scraper package
|
|
18
|
+
// exposes otpCallback in its public HapoalimOptions type.
|
|
19
|
+
type ScraperCallOptions = {
|
|
20
|
+
isBusiness?: boolean;
|
|
21
|
+
duration?: number;
|
|
22
|
+
otpCallback?: () => Promise<string>;
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
export async function scrapePoalim(
|
|
26
|
+
creds: PoalimCreds,
|
|
27
|
+
dateFrom: Date,
|
|
28
|
+
_dateTo: Date,
|
|
29
|
+
headless: boolean,
|
|
30
|
+
otpManager: OtpManager,
|
|
31
|
+
emit: (msg: ServerMessage) => void,
|
|
32
|
+
): Promise<{
|
|
33
|
+
ils: PoalimIlsPayload[];
|
|
34
|
+
foreign: PoalimForeignPayload[];
|
|
35
|
+
swift: PoalimSwiftPayload[];
|
|
36
|
+
}> {
|
|
37
|
+
const otpCallback = () => otpManager.waitForOtp(creds.id, emit, OTP_TIMEOUT_MS);
|
|
38
|
+
|
|
39
|
+
const { hapoalim: hapoalimFn, close } = await init({ headless });
|
|
40
|
+
|
|
41
|
+
type HapoalimFn = (
|
|
42
|
+
c: { userCode: string; password: string },
|
|
43
|
+
o?: ScraperCallOptions,
|
|
44
|
+
) => ReturnType<typeof hapoalimFn>;
|
|
45
|
+
|
|
46
|
+
const scrape = hapoalimFn as unknown as HapoalimFn;
|
|
47
|
+
|
|
48
|
+
// calculate duration based ondateFrom and today. return number of months between the two dates, with a minimum of 1 month
|
|
49
|
+
const months = Math.max(
|
|
50
|
+
1,
|
|
51
|
+
(_dateTo.getFullYear() - dateFrom.getFullYear()) * 12 +
|
|
52
|
+
(_dateTo.getMonth() - dateFrom.getMonth()) +
|
|
53
|
+
1,
|
|
54
|
+
);
|
|
55
|
+
|
|
56
|
+
try {
|
|
57
|
+
const scraper = await scrape(
|
|
58
|
+
{ userCode: creds.userCode, password: creds.password },
|
|
59
|
+
{ isBusiness: creds.options?.isBusinessAccount ?? true, otpCallback, duration: months },
|
|
60
|
+
);
|
|
61
|
+
|
|
62
|
+
if (scraper === 'Unknown Error') {
|
|
63
|
+
throw new Error('Hapoalim login failed: Unknown Error');
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
const { data: accounts } = await scraper.getAccountsData();
|
|
67
|
+
if (!accounts || accounts.length === 0) {
|
|
68
|
+
return { ils: [], foreign: [], swift: [] };
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
const ils: PoalimIlsPayload[] = [];
|
|
72
|
+
const foreign: PoalimForeignPayload[] = [];
|
|
73
|
+
const swift: PoalimSwiftPayload[] = [];
|
|
74
|
+
const isBusiness = creds.options?.isBusinessAccount ?? true;
|
|
75
|
+
|
|
76
|
+
for (const account of accounts) {
|
|
77
|
+
const accountRef = {
|
|
78
|
+
bankNumber: account.bankNumber,
|
|
79
|
+
branchNumber: account.branchNumber,
|
|
80
|
+
accountNumber: account.accountNumber,
|
|
81
|
+
};
|
|
82
|
+
|
|
83
|
+
const { data: ilsData } = await scraper.getILSTransactions(accountRef);
|
|
84
|
+
if (ilsData) {
|
|
85
|
+
ils.push(validatePayload('poalim-ils', ilsData));
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
const { data: foreignData } = await scraper.getForeignTransactions(accountRef, isBusiness);
|
|
89
|
+
if (foreignData) {
|
|
90
|
+
foreign.push(validatePayload('poalim-foreign', foreignData));
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
const { data: swiftData } = await scraper.getForeignSwiftTransactions(accountRef);
|
|
94
|
+
if (swiftData) {
|
|
95
|
+
swift.push(validatePayload('poalim-swift', swiftData));
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
return { ils, foreign, swift };
|
|
100
|
+
} finally {
|
|
101
|
+
await close();
|
|
102
|
+
}
|
|
103
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { FastifyInstance } from 'fastify';
|
|
2
|
+
import { getVault, isLocked, updateVault } from './vault-store.js';
|
|
3
|
+
import { SettingsSchema, type Settings } from './vault.js';
|
|
4
|
+
|
|
5
|
+
function guardLocked(reply: { status(code: number): { send(body: unknown): unknown } }) {
|
|
6
|
+
if (isLocked()) return reply.status(401).send({ error: 'vault-locked' });
|
|
7
|
+
return null;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export async function registerSettingsRoutes(app: FastifyInstance): Promise<void> {
|
|
11
|
+
app.get('/api/vault/settings', async (_req, reply) => {
|
|
12
|
+
const blocked = guardLocked(reply);
|
|
13
|
+
if (blocked) return blocked;
|
|
14
|
+
return getVault().settings;
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
app.put<{ Body: Partial<Settings> }>('/api/vault/settings', async (req, reply) => {
|
|
18
|
+
const blocked = guardLocked(reply);
|
|
19
|
+
if (blocked) return blocked;
|
|
20
|
+
|
|
21
|
+
const parsed = SettingsSchema.partial().safeParse(req.body);
|
|
22
|
+
if (!parsed.success) return reply.status(400).send({ error: parsed.error.issues });
|
|
23
|
+
|
|
24
|
+
await updateVault(v => ({ ...v, settings: { ...v.settings, ...parsed.data } }));
|
|
25
|
+
return getVault().settings;
|
|
26
|
+
});
|
|
27
|
+
}
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import type { FastifyInstance } from 'fastify';
|
|
3
|
+
import { z } from 'zod';
|
|
4
|
+
import { getVault, isLocked, updateVault } from './vault-store.js';
|
|
5
|
+
import {
|
|
6
|
+
CalAccountSchema,
|
|
7
|
+
DiscountAccountSchema,
|
|
8
|
+
IsracardAmexAccountSchema,
|
|
9
|
+
MaxAccountSchema,
|
|
10
|
+
PoalimAccountSchema,
|
|
11
|
+
type Vault,
|
|
12
|
+
} from './vault.js';
|
|
13
|
+
|
|
14
|
+
const SourceConfigSchema = z.discriminatedUnion('type', [
|
|
15
|
+
PoalimAccountSchema.extend({ type: z.literal('poalim') }),
|
|
16
|
+
DiscountAccountSchema.extend({ type: z.literal('discount') }),
|
|
17
|
+
IsracardAmexAccountSchema.extend({ type: z.literal('isracard') }),
|
|
18
|
+
IsracardAmexAccountSchema.extend({ type: z.literal('amex') }),
|
|
19
|
+
CalAccountSchema.extend({ type: z.literal('cal') }),
|
|
20
|
+
MaxAccountSchema.extend({ type: z.literal('max') }),
|
|
21
|
+
]);
|
|
22
|
+
|
|
23
|
+
export type SourceConfig = z.infer<typeof SourceConfigSchema>;
|
|
24
|
+
type SourceType = SourceConfig['type'];
|
|
25
|
+
|
|
26
|
+
function collectSources(vault: Vault): SourceConfig[] {
|
|
27
|
+
return [
|
|
28
|
+
...vault.poalimAccounts.map(a => ({ ...a, type: 'poalim' as const })),
|
|
29
|
+
...vault.discountAccounts.map(a => ({ ...a, type: 'discount' as const })),
|
|
30
|
+
...vault.isracardAccounts.map(a => ({ ...a, type: 'isracard' as const })),
|
|
31
|
+
...vault.amexAccounts.map(a => ({ ...a, type: 'amex' as const })),
|
|
32
|
+
...vault.calAccounts.map(a => ({ ...a, type: 'cal' as const })),
|
|
33
|
+
...vault.maxAccounts.map(a => ({ ...a, type: 'max' as const })),
|
|
34
|
+
];
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function appendSource(vault: Vault, source: SourceConfig): Vault {
|
|
38
|
+
const { type, ...rest } = source;
|
|
39
|
+
const v = { ...vault };
|
|
40
|
+
switch (type) {
|
|
41
|
+
case 'poalim':
|
|
42
|
+
v.poalimAccounts = [...vault.poalimAccounts, rest as (typeof vault.poalimAccounts)[number]];
|
|
43
|
+
break;
|
|
44
|
+
case 'discount':
|
|
45
|
+
v.discountAccounts = [
|
|
46
|
+
...vault.discountAccounts,
|
|
47
|
+
rest as (typeof vault.discountAccounts)[number],
|
|
48
|
+
];
|
|
49
|
+
break;
|
|
50
|
+
case 'isracard':
|
|
51
|
+
v.isracardAccounts = [
|
|
52
|
+
...vault.isracardAccounts,
|
|
53
|
+
rest as (typeof vault.isracardAccounts)[number],
|
|
54
|
+
];
|
|
55
|
+
break;
|
|
56
|
+
case 'amex':
|
|
57
|
+
v.amexAccounts = [...vault.amexAccounts, rest as (typeof vault.amexAccounts)[number]];
|
|
58
|
+
break;
|
|
59
|
+
case 'cal':
|
|
60
|
+
v.calAccounts = [...vault.calAccounts, rest as (typeof vault.calAccounts)[number]];
|
|
61
|
+
break;
|
|
62
|
+
case 'max':
|
|
63
|
+
v.maxAccounts = [...vault.maxAccounts, rest as (typeof vault.maxAccounts)[number]];
|
|
64
|
+
break;
|
|
65
|
+
}
|
|
66
|
+
return v;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
type ArrayKey = Extract<
|
|
70
|
+
keyof Vault,
|
|
71
|
+
| 'poalimAccounts'
|
|
72
|
+
| 'discountAccounts'
|
|
73
|
+
| 'isracardAccounts'
|
|
74
|
+
| 'amexAccounts'
|
|
75
|
+
| 'calAccounts'
|
|
76
|
+
| 'maxAccounts'
|
|
77
|
+
>;
|
|
78
|
+
|
|
79
|
+
const typeToKey: Record<SourceType, ArrayKey> = {
|
|
80
|
+
poalim: 'poalimAccounts',
|
|
81
|
+
discount: 'discountAccounts',
|
|
82
|
+
isracard: 'isracardAccounts',
|
|
83
|
+
amex: 'amexAccounts',
|
|
84
|
+
cal: 'calAccounts',
|
|
85
|
+
max: 'maxAccounts',
|
|
86
|
+
};
|
|
87
|
+
|
|
88
|
+
const typeToPatchSchema: Record<SourceType, z.ZodType> = {
|
|
89
|
+
poalim: PoalimAccountSchema.omit({ id: true }).partial(),
|
|
90
|
+
discount: DiscountAccountSchema.omit({ id: true }).partial(),
|
|
91
|
+
isracard: IsracardAmexAccountSchema.omit({ id: true }).partial(),
|
|
92
|
+
amex: IsracardAmexAccountSchema.omit({ id: true }).partial(),
|
|
93
|
+
cal: CalAccountSchema.omit({ id: true }).partial(),
|
|
94
|
+
max: MaxAccountSchema.omit({ id: true }).partial(),
|
|
95
|
+
};
|
|
96
|
+
|
|
97
|
+
function findSourceKey(
|
|
98
|
+
vault: Vault,
|
|
99
|
+
id: string,
|
|
100
|
+
): { key: ArrayKey; idx: number; type: SourceType } | null {
|
|
101
|
+
for (const [type, key] of Object.entries(typeToKey) as [SourceType, ArrayKey][]) {
|
|
102
|
+
const arr = vault[key] as Array<{ id: string }>;
|
|
103
|
+
const idx = arr.findIndex(a => a.id === id);
|
|
104
|
+
if (idx !== -1) return { key, idx, type };
|
|
105
|
+
}
|
|
106
|
+
return null;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
function patchSource(
|
|
110
|
+
vault: Vault,
|
|
111
|
+
found: { key: ArrayKey; idx: number },
|
|
112
|
+
patch: Record<string, unknown>,
|
|
113
|
+
): Vault {
|
|
114
|
+
const { key, idx } = found;
|
|
115
|
+
const arr = vault[key] as Array<{ id: string }>;
|
|
116
|
+
const newArr = [...arr];
|
|
117
|
+
newArr[idx] = { ...arr[idx], ...patch } as (typeof newArr)[number];
|
|
118
|
+
return { ...vault, [key]: newArr };
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function removeSource(vault: Vault, id: string): Vault | null {
|
|
122
|
+
const found = findSourceKey(vault, id);
|
|
123
|
+
if (!found) return null;
|
|
124
|
+
|
|
125
|
+
const { key } = found;
|
|
126
|
+
const arr = vault[key] as Array<{ id: string }>;
|
|
127
|
+
return { ...vault, [key]: arr.filter(a => a.id !== id) };
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
function guardLocked(reply: { status(code: number): { send(body: unknown): unknown } }) {
|
|
131
|
+
if (isLocked()) {
|
|
132
|
+
return reply.status(401).send({ error: 'vault-locked' });
|
|
133
|
+
}
|
|
134
|
+
return null;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
export async function registerSourcesRoutes(app: FastifyInstance): Promise<void> {
|
|
138
|
+
app.get('/api/vault/sources', async (_req, reply) => {
|
|
139
|
+
const blocked = guardLocked(reply);
|
|
140
|
+
if (blocked) return blocked;
|
|
141
|
+
return collectSources(getVault());
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
app.post<{ Body: SourceConfig }>('/api/vault/sources', async (req, reply) => {
|
|
145
|
+
const blocked = guardLocked(reply);
|
|
146
|
+
if (blocked) return blocked;
|
|
147
|
+
|
|
148
|
+
const parsed = SourceConfigSchema.safeParse({ ...req.body, id: randomUUID() });
|
|
149
|
+
if (!parsed.success) return reply.status(400).send({ error: parsed.error.issues });
|
|
150
|
+
|
|
151
|
+
await updateVault(v => appendSource(v, parsed.data));
|
|
152
|
+
return collectSources(getVault());
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
app.put<{ Params: { id: string }; Body: unknown }>(
|
|
156
|
+
'/api/vault/sources/:id',
|
|
157
|
+
async (req, reply) => {
|
|
158
|
+
const blocked = guardLocked(reply);
|
|
159
|
+
if (blocked) return blocked;
|
|
160
|
+
|
|
161
|
+
const found = findSourceKey(getVault(), req.params.id);
|
|
162
|
+
if (!found) return reply.status(404).send({ error: 'not-found' });
|
|
163
|
+
|
|
164
|
+
const parsed = typeToPatchSchema[found.type].safeParse(req.body);
|
|
165
|
+
if (!parsed.success) return reply.status(400).send({ error: parsed.error.issues });
|
|
166
|
+
|
|
167
|
+
await updateVault(v => patchSource(v, found, parsed.data as Record<string, unknown>));
|
|
168
|
+
return collectSources(getVault());
|
|
169
|
+
},
|
|
170
|
+
);
|
|
171
|
+
|
|
172
|
+
app.delete<{ Params: { id: string } }>('/api/vault/sources/:id', async (req, reply) => {
|
|
173
|
+
const blocked = guardLocked(reply);
|
|
174
|
+
if (blocked) return blocked;
|
|
175
|
+
|
|
176
|
+
const updated = removeSource(getVault(), req.params.id);
|
|
177
|
+
if (!updated) return reply.status(404).send({ error: 'not-found' });
|
|
178
|
+
|
|
179
|
+
await updateVault(() => updated);
|
|
180
|
+
return collectSources(getVault());
|
|
181
|
+
});
|
|
182
|
+
}
|