northbid 0.0.0-stage → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,254 @@
1
+ // Query construction over the normalized tables. Free text goes through FTS5
2
+ // (diacritic-insensitive, both languages at once); everything else is plain SQL.
3
+
4
+ import { TABLES } from './datasets.js';
5
+
6
+ /** Raised for problems the caller can fix by changing the arguments. */
7
+ export class InputError extends Error {}
8
+
9
+ /**
10
+ * Turns user text into an FTS5 expression. Bare words and "quoted phrases"
11
+ * become quoted terms, `word*` is a prefix match, and an uppercase OR between
12
+ * two terms is kept. Everything else is literal, so no input is a syntax error.
13
+ * @param {string} text
14
+ * @param {'all' | 'any'} [match] how terms without an explicit OR combine
15
+ * @returns {string | null} null when the text has nothing searchable
16
+ */
17
+ export function ftsQuery(text, match = 'all') {
18
+ const parts = [];
19
+ let or = false;
20
+ for (const m of text.matchAll(/"([^"]*)"|(\S+)/g)) {
21
+ if (m[2] === 'OR') { or = parts.length > 0; continue; }
22
+ if (m[2] === 'AND') continue;
23
+ const prefix = m[2] !== undefined && m[2].endsWith('*');
24
+ const raw = (m[1] ?? m[2]).replace(/["*]/g, ' ').trim();
25
+ if (!/[\p{L}\p{N}]/u.test(raw)) continue;
26
+ if (parts.length) parts.push(or || match === 'any' ? 'OR' : 'AND');
27
+ parts.push(`"${raw}"${prefix ? '*' : ''}`);
28
+ or = false;
29
+ }
30
+ return parts.length ? parts.join(' ') : null;
31
+ }
32
+
33
+ // bm25 weights, in TABLES[...].fts column order.
34
+ const WEIGHTS = {
35
+ tenders: [10, 10, 1, 1, 4, 4, 4, 4, 2, 2, 2, 2, 6],
36
+ standing_offers: [8, 8, 4, 4, 6, 6, 6, 2, 2, 6],
37
+ awards: [10, 10, 1, 1, 4, 4, 6, 2, 2, 2, 2, 6, 6],
38
+ };
39
+
40
+ function required(text, name, match = 'all') {
41
+ const q = ftsQuery(text, match);
42
+ if (!q) throw new InputError(`"${name}" has no searchable words; use letters or digits.`);
43
+ return q;
44
+ }
45
+
46
+ // A value stored as one line of a newline-separated column.
47
+ const hasLine = (col) => `instr(char(10) || ${col} || char(10), char(10) || ? || char(10)) > 0`;
48
+ // A code prefix at the start of any line.
49
+ const hasLinePrefix = (col) => `instr(char(10) || ${col}, char(10) || ?) > 0`;
50
+ const likeEither = (col) => `(${col}_en LIKE ? ESCAPE '\\' OR ${col}_fr LIKE ? ESCAPE '\\')`;
51
+ const like = (s) => `%${s.replace(/[\\%_]/g, '\\$&')}%`;
52
+
53
+ // UNSPSC codes are four two-digit levels padded with zeros: 43000000 is a
54
+ // segment, 43230000 a family. Dropping the padding gives the prefix to match.
55
+ function unspscPrefix(code) {
56
+ const prefix = code.replace(/(00)+$/, '');
57
+ if (!prefix) throw new InputError('"unspsc" must not be all zeros.');
58
+ return prefix;
59
+ }
60
+
61
+ // The FROM and WHERE of a filtered query over `table`, aliased `t`.
62
+ function scope(table, { match, where, params }) {
63
+ const fts = `${table}_fts`;
64
+ const from = match.length
65
+ ? `${fts} JOIN ${table} t ON t.rowid = ${fts}.rowid WHERE ${fts} MATCH ?`
66
+ : `${table} t WHERE 1`;
67
+ return {
68
+ sql: from + where.map((w) => ` AND ${w}`).join(''),
69
+ args: match.length ? [match.join(' AND '), ...params] : params,
70
+ };
71
+ }
72
+
73
+ function run(db, table, { match, where, params, order, limit, offset }) {
74
+ const { sql, args } = scope(table, { match, where, params });
75
+ const total = db.prepare(`SELECT count(*) AS n FROM ${sql}`).get(...args).n;
76
+ const rows = db.prepare(`SELECT t.* FROM ${sql} ORDER BY ${order} LIMIT ? OFFSET ?`).all(...args, limit, offset);
77
+ return { total, rows };
78
+ }
79
+
80
+ /** @param {import('node:sqlite').DatabaseSync} db */
81
+ export function searchTenders(db, a) {
82
+ const match = [];
83
+ const where = [];
84
+ const params = [];
85
+ if (a.query) match.push(`(${required(a.query, 'query', a.match)})`);
86
+ if (a.buyer) match.push(`{buyer_en buyer_fr end_user_en end_user_fr} : (${required(a.buyer, 'buyer')})`);
87
+ if (a.category?.length) {
88
+ where.push(`(${a.category.map(() => hasLine('t.categories')).join(' OR ')})`);
89
+ params.push(...a.category);
90
+ }
91
+ if (a.unspsc) {
92
+ where.push(hasLinePrefix('t.unspsc_codes'));
93
+ params.push(unspscPrefix(a.unspsc));
94
+ }
95
+ if (a.region) {
96
+ where.push(likeEither('t.regions_of_delivery'));
97
+ params.push(like(a.region), like(a.region));
98
+ }
99
+ if (a.notice_type) {
100
+ where.push(likeEither('t.notice_type'));
101
+ params.push(like(a.notice_type), like(a.notice_type));
102
+ }
103
+ if (a.closing_after) { where.push('t.closing_date >= ?'); params.push(a.closing_after); }
104
+ // Closing timestamps sort after their own date, so compare against the day's end.
105
+ if (a.closing_before) { where.push('t.closing_date <= ?'); params.push(`${a.closing_before}T23:59:59-05:00`); }
106
+ if (a.published_after) { where.push('t.publication_date >= ?'); params.push(a.published_after); }
107
+ if (!a.include_past_deadline) { where.push('t.closing_date >= ?'); params.push(a.today); }
108
+
109
+ const sort = a.sort ?? (a.query ? 'relevance' : 'closing_soonest');
110
+ const order = {
111
+ relevance: a.query ? `bm25(tenders_fts, ${WEIGHTS.tenders.join(', ')})` : 't.closing_date',
112
+ closing_soonest: 't.closing_date',
113
+ newest: 't.publication_date DESC',
114
+ }[sort];
115
+ return run(db, 'tenders', {
116
+ match, where, params, order: `${order}, t.reference_number`, limit: a.limit, offset: a.offset,
117
+ });
118
+ }
119
+
120
+ /** Notices whose reference or solicitation number equals `id`, newest amendment first. */
121
+ export function getTenders(db, id) {
122
+ return db
123
+ .prepare(
124
+ `SELECT * FROM tenders WHERE reference_number = ?1 COLLATE NOCASE OR solicitation_number = ?1 COLLATE NOCASE
125
+ ORDER BY publication_date DESC LIMIT 20`,
126
+ )
127
+ .all(id.trim());
128
+ }
129
+
130
+ /** @param {import('node:sqlite').DatabaseSync} db */
131
+ export function searchStandingOffers(db, a, today) {
132
+ const match = [];
133
+ const where = [];
134
+ const params = [];
135
+ if (a.query) match.push(`(${required(a.query, 'query', a.match)})`);
136
+ if (a.supplier) {
137
+ match.push(`{supplier_name supplier_legal_name supplier_operating_name} : (${required(a.supplier, 'supplier')})`);
138
+ }
139
+ if (a.department) match.push(`{department_en department_fr} : (${required(a.department, 'department')})`);
140
+ if (a.commodity) {
141
+ where.push(hasLinePrefix('t.commodity_codes'));
142
+ params.push(unspscPrefix(a.commodity));
143
+ }
144
+ if (a.agreement_type) { where.push('t.agreement_type_en = ? COLLATE NOCASE'); params.push(a.agreement_type); }
145
+ if (a.region) {
146
+ where.push(likeEither('t.delivery_points'));
147
+ params.push(like(a.region), like(a.region));
148
+ }
149
+ if (!a.include_expired) { where.push('t.expiry_date >= ?'); params.push(today); }
150
+ const order = a.query ? `bm25(standing_offers_fts, ${WEIGHTS.standing_offers.join(', ')})` : 't.expiry_date DESC';
151
+ return run(db, 'standing_offers', {
152
+ match, where, params, order: `${order}, t.agreement_number`, limit: a.limit, offset: a.offset,
153
+ });
154
+ }
155
+
156
+ function awardFilters(a) {
157
+ const match = [];
158
+ const where = [];
159
+ const params = [];
160
+ if (a.query) match.push(`(${required(a.query, 'query', a.match)})`);
161
+ if (a.supplier) match.push(`{supplier_name} : (${required(a.supplier, 'supplier')})`);
162
+ if (a.buyer) match.push(`{buyer_en buyer_fr end_user_en end_user_fr} : (${required(a.buyer, 'buyer')})`);
163
+ if (a.category?.length) {
164
+ where.push(`(${a.category.map(() => hasLine('t.categories')).join(' OR ')})`);
165
+ params.push(...a.category);
166
+ }
167
+ if (a.unspsc) {
168
+ where.push(hasLinePrefix('t.unspsc_codes'));
169
+ params.push(unspscPrefix(a.unspsc));
170
+ }
171
+ if (a.region) {
172
+ where.push(likeEither('t.regions_of_delivery'));
173
+ params.push(like(a.region), like(a.region));
174
+ }
175
+ // "Non-competitive" contains "Competitive", so match on the start of the value.
176
+ if (a.competitive === true) where.push(`t.procurement_method_en LIKE 'Competitive%'`);
177
+ if (a.competitive === false) {
178
+ where.push(`t.procurement_method_en IN ('Non-competitive', 'Advance contract award notice')`);
179
+ }
180
+ if (a.awarded_after) { where.push('t.award_date >= ?'); params.push(a.awarded_after); }
181
+ if (a.awarded_before) { where.push('t.award_date <= ?'); params.push(a.awarded_before); }
182
+ if (a.min_value !== undefined) { where.push('t.value >= ?'); params.push(a.min_value); }
183
+ if (a.max_value !== undefined) { where.push('t.value > 0 AND t.value <= ?'); params.push(a.max_value); }
184
+ if (!a.include_cancelled) where.push(`t.status_en <> 'Cancelled'`);
185
+ return { match, where, params };
186
+ }
187
+
188
+ /** @param {import('node:sqlite').DatabaseSync} db */
189
+ export function searchAwards(db, a) {
190
+ const sort = a.sort ?? (a.query ? 'relevance' : 'newest');
191
+ const order = {
192
+ relevance: a.query ? `bm25(awards_fts, ${WEIGHTS.awards.join(', ')})` : 't.award_date DESC',
193
+ newest: 't.award_date DESC',
194
+ highest_value: 't.value DESC',
195
+ }[sort];
196
+ return run(db, 'awards', {
197
+ ...awardFilters(a), order: `${order}, t.reference_number`, limit: a.limit, offset: a.offset,
198
+ });
199
+ }
200
+
201
+ /** Award notices whose reference, solicitation or contract number equals `id`, newest first. */
202
+ export function getAwards(db, id, limit) {
203
+ const sql = `FROM awards WHERE reference_number = ?1 COLLATE NOCASE OR solicitation_number = ?1 COLLATE NOCASE
204
+ OR contract_number = ?1 COLLATE NOCASE`;
205
+ const key = id.trim();
206
+ return {
207
+ total: db.prepare(`SELECT count(*) AS n ${sql}`).get(key).n,
208
+ rows: db.prepare(`SELECT * ${sql} ORDER BY award_date DESC, reference_number LIMIT ?2`).all(key, limit),
209
+ };
210
+ }
211
+
212
+ // Supplier names are grouped ignoring case, periods and commas ("Acme Inc."
213
+ // and "ACME INC"); other spelling variants of one company stay separate.
214
+ // Only Canadian-dollar values are summed.
215
+ const GROUPS = {
216
+ supplier: {
217
+ key: `replace(replace(upper(t.supplier_name), '.', ''), ',', '')`,
218
+ name: () => 'max(t.supplier_name)',
219
+ },
220
+ buyer: { key: 't.buyer_en', name: (lang) => (lang === 'fr' ? `max(coalesce(nullif(t.buyer_fr, ''), t.buyer_en))` : 't.buyer_en') },
221
+ year: { key: 'substr(t.award_date, 1, 4)', name: () => 'substr(t.award_date, 1, 4)' },
222
+ };
223
+
224
+ /**
225
+ * Award counts and Canadian-dollar totals per supplier, buyer or award year.
226
+ * @param {import('node:sqlite').DatabaseSync} db
227
+ */
228
+ export function summarizeAwards(db, a) {
229
+ const group = GROUPS[a.group_by];
230
+ const { sql, args } = scope('awards', awardFilters(a));
231
+ const cad = `CASE WHEN t.currency = 'CAD' THEN t.value ELSE 0 END`;
232
+ const order = { total_value: 'total_value_cad DESC, awards DESC', awards: 'awards DESC, total_value_cad DESC' }[a.sort];
233
+ const overall = db
234
+ .prepare(
235
+ `SELECT count(*) AS awards, count(DISTINCT ${group.key}) AS groups, coalesce(sum(${cad}), 0) AS total_value_cad,
236
+ coalesce(sum(t.value = 0), 0) AS without_value,
237
+ coalesce(sum(t.value > 0 AND t.currency <> 'CAD'), 0) AS other_currency
238
+ FROM ${sql}`,
239
+ )
240
+ .get(...args);
241
+ const rows = db
242
+ .prepare(
243
+ `SELECT ${group.name(a.lang)} AS name, count(*) AS awards, sum(${cad}) AS total_value_cad,
244
+ max(${cad}) AS largest_value_cad, min(t.award_date) AS first_award, max(t.award_date) AS latest_award
245
+ FROM ${sql} GROUP BY ${group.key} ORDER BY ${a.group_by === 'year' ? 'name DESC' : order}, name
246
+ LIMIT ? OFFSET ?`,
247
+ )
248
+ .all(...args, a.limit, a.offset);
249
+ return { overall, rows };
250
+ }
251
+
252
+ for (const [table, weights] of Object.entries(WEIGHTS)) {
253
+ if (weights.length !== TABLES[table].fts.length) throw new Error(`bm25 weights out of step with ${table} fts columns`);
254
+ }
@@ -0,0 +1,213 @@
1
+ // Local SQLite cache of the source files. Each file is fetched with a
2
+ // conditional request and loaded atomically; the database is disposable and
3
+ // is rebuilt whenever the schema version changes.
4
+
5
+ import { mkdirSync } from 'node:fs';
6
+ import { homedir } from 'node:os';
7
+ import { join } from 'node:path';
8
+ import { DatabaseSync } from 'node:sqlite';
9
+ import { records } from './csv.js';
10
+ import { FILES, TABLES } from './datasets.js';
11
+ import { download } from './download.js';
12
+
13
+ const SCHEMA_VERSION = 2;
14
+
15
+ /** Cache directory: explicit override, then the plugin data dir, then the OS cache dir. */
16
+ export function defaultDataDir(env = process.env) {
17
+ if (env.NORTHBID_DATA_DIR) return env.NORTHBID_DATA_DIR;
18
+ if (env.CLAUDE_PLUGIN_DATA) return env.CLAUDE_PLUGIN_DATA;
19
+ if (process.platform === 'win32' && env.LOCALAPPDATA) return join(env.LOCALAPPDATA, 'northbid');
20
+ if (process.platform === 'darwin') return join(homedir(), 'Library', 'Caches', 'northbid');
21
+ return join(env.XDG_CACHE_HOME || join(homedir(), '.cache'), 'northbid');
22
+ }
23
+
24
+ export class Store {
25
+ /**
26
+ * @param {string} dir cache directory, or ':memory:'
27
+ * @param {{ offline?: boolean, userAgent?: string }} [options]
28
+ */
29
+ constructor(dir, { offline = false, userAgent = 'northbid' } = {}) {
30
+ if (dir === ':memory:') {
31
+ this.path = dir;
32
+ } else {
33
+ mkdirSync(dir, { recursive: true });
34
+ this.path = join(dir, 'northbid.sqlite');
35
+ }
36
+ this.offline = offline;
37
+ this.userAgent = userAgent;
38
+ this.inflight = new Map();
39
+ this.db = new DatabaseSync(this.path);
40
+ this.db.exec('PRAGMA journal_mode = WAL; PRAGMA busy_timeout = 10000;');
41
+ if (this.db.prepare('PRAGMA user_version').get().user_version !== SCHEMA_VERSION) this.#createSchema();
42
+ }
43
+
44
+ #createSchema() {
45
+ const sql = ['DROP TABLE IF EXISTS files'];
46
+ for (const [name, t] of Object.entries(TABLES)) {
47
+ sql.push(`DROP TABLE IF EXISTS ${name}_fts`, `DROP TABLE IF EXISTS ${name}`);
48
+ const cols = t.columns.map(
49
+ (c) => `${c} ${t.real?.includes(c) ? 'REAL' : 'TEXT'} NOT NULL${c === t.primaryKey ? ' PRIMARY KEY' : ''}`,
50
+ );
51
+ sql.push(`CREATE TABLE ${name} (${cols.join(', ')}, source TEXT NOT NULL)`);
52
+ for (const c of t.indexes) sql.push(`CREATE INDEX ${name}_${c} ON ${name} (${c})`);
53
+ sql.push(
54
+ `CREATE VIRTUAL TABLE ${name}_fts USING fts5(${t.fts.join(', ')}, content='${name}', ` +
55
+ `content_rowid='rowid', tokenize='unicode61 remove_diacritics 2')`,
56
+ );
57
+ }
58
+ sql.push(
59
+ `CREATE TABLE files (id TEXT PRIMARY KEY, etag TEXT, source_updated TEXT, fetched_at TEXT NOT NULL,
60
+ checked_at TEXT NOT NULL, row_count INTEGER NOT NULL)`,
61
+ `PRAGMA user_version = ${SCHEMA_VERSION}`,
62
+ );
63
+ this.db.exec(sql.join(';\n'));
64
+ }
65
+
66
+ #file(id) {
67
+ return this.db.prepare('SELECT * FROM files WHERE id = ?').get(id);
68
+ }
69
+
70
+ /**
71
+ * Replaces everything previously loaded from `file` with `rows`.
72
+ * @param {(typeof FILES)[number]} file
73
+ * @param {Record<string, string>[]} rows
74
+ * @param {{ etag?: string | null, sourceUpdated?: string | null }} [info]
75
+ */
76
+ load(file, rows, { etag = null, sourceUpdated = null } = {}) {
77
+ const { columns } = TABLES[file.table];
78
+ const now = new Date().toISOString();
79
+ const insert = this.db.prepare(
80
+ `INSERT OR ${file.onConflict} INTO ${file.table} (${columns.join(', ')}, source)
81
+ VALUES (${columns.map(() => '?').join(', ')}, ?)`,
82
+ );
83
+ this.db.exec('BEGIN IMMEDIATE');
84
+ try {
85
+ this.db.prepare(`DELETE FROM ${file.table} WHERE source = ?`).run(file.id);
86
+ for (const row of rows) insert.run(...columns.map((c) => row[c] ?? ''), file.id);
87
+ this.db.exec(`INSERT INTO ${file.table}_fts(${file.table}_fts) VALUES('rebuild')`);
88
+ this.db
89
+ .prepare(
90
+ `INSERT INTO files (id, etag, source_updated, fetched_at, checked_at, row_count) VALUES (?, ?, ?, ?, ?, ?)
91
+ ON CONFLICT(id) DO UPDATE SET etag = excluded.etag, source_updated = excluded.source_updated,
92
+ fetched_at = excluded.fetched_at, checked_at = excluded.checked_at, row_count = excluded.row_count`,
93
+ )
94
+ .run(file.id, etag, sourceUpdated, now, now, rows.length);
95
+ this.db.exec('COMMIT');
96
+ } catch (err) {
97
+ this.db.exec('ROLLBACK');
98
+ throw err;
99
+ }
100
+ }
101
+
102
+ async #download(file) {
103
+ const known = this.#file(file.id);
104
+ const headers = { 'user-agent': this.userAgent };
105
+ if (known?.etag) headers['if-none-match'] = known.etag;
106
+ if (known?.source_updated) headers['if-modified-since'] = new Date(known.source_updated).toUTCString();
107
+ const res = await download(file.url, headers);
108
+ if (res.statusCode === 304) {
109
+ res.resume();
110
+ this.db.prepare('UPDATE files SET checked_at = ? WHERE id = ?').run(new Date().toISOString(), file.id);
111
+ return;
112
+ }
113
+ if (res.statusCode !== 200) {
114
+ res.resume();
115
+ throw new Error(`HTTP ${res.statusCode}`);
116
+ }
117
+ const rows = [];
118
+ for await (const row of file.rows(records(res.setEncoding('utf8')))) rows.push(row);
119
+ // An empty required file is an upstream fault; keep what we have.
120
+ if (rows.length === 0 && file.required) throw new Error('the source file contained no rows');
121
+ const modified = Date.parse(res.headers['last-modified'] ?? '');
122
+ this.load(file, rows, {
123
+ etag: res.headers.etag ?? null,
124
+ sourceUpdated: Number.isNaN(modified) ? null : new Date(modified).toISOString().replace('.000Z', 'Z'),
125
+ });
126
+ }
127
+
128
+ /**
129
+ * Brings the files behind `table` up to date and returns warnings for
130
+ * anything that could not be refreshed. Throws only when a required file
131
+ * has never been loaded and cannot be fetched.
132
+ * @param {string | null} table null for every table
133
+ * @param {{ force?: boolean }} [options] force ignores the per-file TTL
134
+ * @returns {Promise<string[]>}
135
+ */
136
+ async refresh(table = null, { force = false } = {}) {
137
+ const warnings = [];
138
+ const due = FILES.filter((f) => table === null || f.table === table);
139
+ await Promise.all(
140
+ due.map(async (file) => {
141
+ const known = this.#file(file.id);
142
+ if (this.offline) {
143
+ if (!known && file.required) {
144
+ throw new Error(`${file.label} has not been downloaded yet and NORTHBID_OFFLINE is set.`);
145
+ }
146
+ return;
147
+ }
148
+ if (!force && known && Date.now() - Date.parse(known.checked_at) < file.ttl) return;
149
+ let job = this.inflight.get(file.id);
150
+ if (!job) {
151
+ job = this.#download(file).finally(() => this.inflight.delete(file.id));
152
+ this.inflight.set(file.id, job);
153
+ }
154
+ try {
155
+ await job;
156
+ } catch (err) {
157
+ const reason = err.name === 'TimeoutError' ? 'the request timed out' : err.cause?.message || err.message;
158
+ if (!known && file.required) {
159
+ throw new Error(
160
+ `Could not download ${file.label} from ${file.url}: ${reason}. ` +
161
+ 'Check the network connection and call refresh_data to retry.',
162
+ );
163
+ }
164
+ warnings.push(
165
+ known
166
+ ? `Could not refresh ${file.label} (${reason}); showing data fetched ${known.fetched_at}.`
167
+ : `${file.label} is unavailable (${reason}).`,
168
+ );
169
+ }
170
+ }),
171
+ );
172
+ return warnings;
173
+ }
174
+
175
+ /** When the source last published the newest of the main files behind `table`. */
176
+ sourceUpdated(table) {
177
+ let latest = null;
178
+ for (const f of FILES) {
179
+ if (f.table !== table || f.supplementary) continue;
180
+ const updated = this.#file(f.id)?.source_updated;
181
+ if (updated && (!latest || updated > latest)) latest = updated;
182
+ }
183
+ return latest;
184
+ }
185
+
186
+ status() {
187
+ return {
188
+ database: this.path,
189
+ offline: this.offline,
190
+ files: FILES.map((f) => {
191
+ const known = this.#file(f.id);
192
+ return {
193
+ id: f.id,
194
+ label: f.label,
195
+ url: f.url,
196
+ table: f.table,
197
+ loaded: Boolean(known),
198
+ rows_in_file: known?.row_count ?? 0,
199
+ source_updated: known?.source_updated ?? null,
200
+ fetched_at: known?.fetched_at ?? null,
201
+ checked_at: known?.checked_at ?? null,
202
+ };
203
+ }),
204
+ tables: Object.fromEntries(
205
+ Object.keys(TABLES).map((t) => [t, this.db.prepare(`SELECT count(*) AS n FROM ${t}`).get().n]),
206
+ ),
207
+ };
208
+ }
209
+
210
+ close() {
211
+ this.db.close();
212
+ }
213
+ }
package/server/text.js ADDED
@@ -0,0 +1,73 @@
1
+ // Text clean-up for CanadaBuys fields: HTML entities and tags in free text,
2
+ // and the `*value` / newline convention used for multi-value cells.
3
+
4
+ const NAMED = {
5
+ amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ', thinsp: ' ', ensp: ' ', emsp: ' ',
6
+ rsquo: '’', lsquo: '‘', rdquo: '”', ldquo: '“', laquo: '«', raquo: '»', sbquo: '‚', bdquo: '„',
7
+ ndash: '–', mdash: '—', hellip: '…', bull: '•', middot: '·', deg: '°', sup2: '²', sup3: '³',
8
+ frac12: '½', frac14: '¼', frac34: '¾', plusmn: '±', times: '×', divide: '÷', micro: 'µ',
9
+ copy: '©', reg: '®', trade: '™', euro: '€', pound: '£', cent: '¢', sect: '§', para: '¶',
10
+ oelig: 'œ', OElig: 'Œ', aelig: 'æ', AElig: 'Æ', szlig: 'ß', ccedil: 'ç', Ccedil: 'Ç',
11
+ ntilde: 'ñ', Ntilde: 'Ñ', yuml: 'ÿ', Yuml: 'Ÿ', shy: '', zwnj: '', zwj: '',
12
+ };
13
+ // Accented vowels follow a regular naming scheme: &eacute; &Egrave; &ocirc; …
14
+ const ACCENTS = { acute: '\u0301', grave: '\u0300', circ: '\u0302', uml: '\u0308', tilde: '\u0303', ring: '\u030a' };
15
+ for (const letter of 'aeiouyAEIOUY') {
16
+ for (const [name, mark] of Object.entries(ACCENTS)) {
17
+ NAMED[letter + name] ??= (letter + mark).normalize('NFC');
18
+ }
19
+ }
20
+
21
+ /** @param {string} s */
22
+ export function decodeEntities(s) {
23
+ if (!s.includes('&')) return s;
24
+ return s.replace(/&(#x[0-9a-f]+|#\d+|[a-z][a-z0-9]*);/gi, (whole, body) => {
25
+ if (body[0] !== '#') return NAMED[body] ?? whole;
26
+ const code = body[1] === 'x' || body[1] === 'X' ? parseInt(body.slice(2), 16) : parseInt(body.slice(1), 10);
27
+ return code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : whole;
28
+ });
29
+ }
30
+
31
+ /** Free text: strips tags, decodes entities, normalizes whitespace. */
32
+ export function cleanText(s) {
33
+ if (!s) return '';
34
+ return decodeEntities(
35
+ s.replace(/<br\s*\/?>|<\/(?:p|div|li|tr|h[1-6])>/gi, '\n').replace(/<\/?[a-z][^<>]*>/gi, ''),
36
+ )
37
+ .replace(/\r\n?/g, '\n')
38
+ .replace(/[ \t\u00a0]+/g, ' ')
39
+ .replace(/ ?\n ?/g, '\n')
40
+ .replace(/\n{3,}/g, '\n\n')
41
+ .trim();
42
+ }
43
+
44
+ /** Single-value cell. */
45
+ export function trim(s) {
46
+ return s ? decodeEntities(s).trim() : '';
47
+ }
48
+
49
+ /** Multi-value cell (`*A\n*B`) to newline-separated values without the marker. */
50
+ export function multi(s) {
51
+ if (!s) return '';
52
+ return s
53
+ .split('\n')
54
+ .map((v) => decodeEntities(v.replace(/^\s*\*/, '')).trim())
55
+ .filter(Boolean)
56
+ .join('\n');
57
+ }
58
+
59
+ /** Standing-offer cell (`"A; B"` with literal quotes) to newline-separated values. */
60
+ export function quotedList(s) {
61
+ if (!s) return '';
62
+ return s
63
+ .replace(/^\s*"|"\s*$/g, '')
64
+ .split(';')
65
+ .map((v) => v.trim())
66
+ .filter(Boolean)
67
+ .join('\n');
68
+ }
69
+
70
+ /** Newline-separated storage form back to an array. */
71
+ export function lines(s) {
72
+ return s ? s.split('\n') : [];
73
+ }