northbid 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.fr.md +130 -0
- package/README.md +126 -2
- package/package.json +32 -4
- package/server/csv.js +88 -0
- package/server/datasets.js +328 -0
- package/server/download.js +91 -0
- package/server/index.js +35 -0
- package/server/mcp.js +98 -0
- package/server/search.js +254 -0
- package/server/store.js +213 -0
- package/server/text.js +73 -0
- package/server/tools.js +598 -0
- package/server/validate.js +57 -0
|
@@ -0,0 +1,328 @@
|
|
|
1
|
+
// Source files, the tables they load into, and the mapping from CanadaBuys
|
|
2
|
+
// column names to the normalized schema. Multi-value columns are stored as
|
|
3
|
+
// newline-separated text; bilingual columns end in _en / _fr.
|
|
4
|
+
|
|
5
|
+
import { cleanText, multi, quotedList, trim } from './text.js';
|
|
6
|
+
|
|
7
|
+
const HOUR = 3_600_000;
|
|
8
|
+
const PUB = 'https://canadabuys.canada.ca/opendata/pub/';
|
|
9
|
+
|
|
10
|
+
export const TABLES = {
|
|
11
|
+
tenders: {
|
|
12
|
+
primaryKey: 'reference_number',
|
|
13
|
+
columns: [
|
|
14
|
+
'reference_number', 'amendment_number', 'solicitation_number',
|
|
15
|
+
'title_en', 'title_fr', 'description_en', 'description_fr',
|
|
16
|
+
'publication_date', 'closing_date', 'amendment_date', 'expected_start_date', 'expected_end_date',
|
|
17
|
+
'status_en', 'status_fr', 'categories',
|
|
18
|
+
'notice_type_en', 'notice_type_fr', 'procurement_method_en', 'procurement_method_fr',
|
|
19
|
+
'selection_criteria_en', 'selection_criteria_fr',
|
|
20
|
+
'limited_tendering_reason_en', 'limited_tendering_reason_fr',
|
|
21
|
+
'trade_agreements_en', 'trade_agreements_fr',
|
|
22
|
+
'regions_of_opportunity_en', 'regions_of_opportunity_fr',
|
|
23
|
+
'regions_of_delivery_en', 'regions_of_delivery_fr',
|
|
24
|
+
'unspsc_codes', 'unspsc_en', 'unspsc_fr', 'gsin_codes', 'gsin_en', 'gsin_fr',
|
|
25
|
+
'buyer_en', 'buyer_fr', 'buyer_address_en', 'buyer_address_fr',
|
|
26
|
+
'end_user_en', 'end_user_fr',
|
|
27
|
+
'contact_name', 'contact_email', 'contact_phone',
|
|
28
|
+
'notice_url_en', 'notice_url_fr', 'attachments_en', 'attachments_fr',
|
|
29
|
+
],
|
|
30
|
+
// Column order here is the bm25 weight order in search.js.
|
|
31
|
+
fts: [
|
|
32
|
+
'title_en', 'title_fr', 'description_en', 'description_fr',
|
|
33
|
+
'unspsc_en', 'unspsc_fr', 'gsin_en', 'gsin_fr',
|
|
34
|
+
'buyer_en', 'buyer_fr', 'end_user_en', 'end_user_fr', 'solicitation_number',
|
|
35
|
+
],
|
|
36
|
+
indexes: ['closing_date', 'publication_date', 'solicitation_number'],
|
|
37
|
+
},
|
|
38
|
+
standing_offers: {
|
|
39
|
+
primaryKey: null,
|
|
40
|
+
columns: [
|
|
41
|
+
'agreement_number', 'agreement_type_en', 'agreement_type_fr', 'title_en', 'title_fr',
|
|
42
|
+
'commodity_codes', 'commodity_en', 'commodity_fr',
|
|
43
|
+
'supplier_name', 'supplier_legal_name', 'supplier_operating_name',
|
|
44
|
+
'department_en', 'department_fr', 'award_date', 'expiry_date',
|
|
45
|
+
'delivery_point_codes', 'delivery_points_en', 'delivery_points_fr',
|
|
46
|
+
],
|
|
47
|
+
fts: [
|
|
48
|
+
'title_en', 'title_fr', 'commodity_en', 'commodity_fr',
|
|
49
|
+
'supplier_name', 'supplier_legal_name', 'supplier_operating_name',
|
|
50
|
+
'department_en', 'department_fr', 'agreement_number',
|
|
51
|
+
],
|
|
52
|
+
indexes: ['expiry_date', 'agreement_number'],
|
|
53
|
+
},
|
|
54
|
+
awards: {
|
|
55
|
+
primaryKey: 'reference_number',
|
|
56
|
+
columns: [
|
|
57
|
+
'reference_number', 'amendment_number', 'solicitation_number', 'contract_number',
|
|
58
|
+
'title_en', 'title_fr', 'description_en', 'description_fr',
|
|
59
|
+
'publication_date', 'award_date', 'amendment_date', 'contract_start_date', 'contract_end_date',
|
|
60
|
+
'value', 'currency', 'status_en', 'status_fr',
|
|
61
|
+
'instrument_type_en', 'instrument_type_fr', 'amendment_type_en', 'amendment_type_fr', 'categories',
|
|
62
|
+
'notice_type_en', 'notice_type_fr', 'procurement_method_en', 'procurement_method_fr',
|
|
63
|
+
'selection_criteria_en', 'selection_criteria_fr',
|
|
64
|
+
'limited_tendering_reason_en', 'limited_tendering_reason_fr',
|
|
65
|
+
'trade_agreements_en', 'trade_agreements_fr',
|
|
66
|
+
'regions_of_delivery_en', 'regions_of_delivery_fr',
|
|
67
|
+
'unspsc_codes', 'unspsc_en', 'unspsc_fr',
|
|
68
|
+
'supplier_name', 'supplier_location_en', 'supplier_location_fr',
|
|
69
|
+
'buyer_en', 'buyer_fr', 'end_user_en', 'end_user_fr',
|
|
70
|
+
'contact_name', 'contact_email', 'contact_phone',
|
|
71
|
+
],
|
|
72
|
+
// Stored as numbers so they can be compared and summed.
|
|
73
|
+
real: ['value'],
|
|
74
|
+
fts: [
|
|
75
|
+
'title_en', 'title_fr', 'description_en', 'description_fr', 'unspsc_en', 'unspsc_fr',
|
|
76
|
+
'supplier_name', 'buyer_en', 'buyer_fr', 'end_user_en', 'end_user_fr',
|
|
77
|
+
'solicitation_number', 'contract_number',
|
|
78
|
+
],
|
|
79
|
+
indexes: ['award_date', 'value', 'solicitation_number', 'contract_number', 'supplier_name'],
|
|
80
|
+
},
|
|
81
|
+
};
|
|
82
|
+
|
|
83
|
+
function address(get, lang) {
|
|
84
|
+
const province = get(`contractingEntityAddressProvince-entiteContractanteAdresseProvince-${lang}`);
|
|
85
|
+
const postal = get('contractingEntityAddressPostalCode-entiteContractanteAdresseCodePostal');
|
|
86
|
+
return [
|
|
87
|
+
get(`contractingEntityAddressLine-ligneAdresseEntiteContractante-${lang}`),
|
|
88
|
+
get(`contractingEntityAddressCity-entiteContractanteAdresseVille-${lang}`),
|
|
89
|
+
[province, postal].map(trim).filter(Boolean).join(' '),
|
|
90
|
+
get(`contractingEntityAddressCountry-entiteContractanteAdressePays-${lang}`),
|
|
91
|
+
].map(trim).filter(Boolean).join(', ');
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// Source timestamps are local to UTC-05:00 year-round and carry no offset.
|
|
95
|
+
function closing(s) {
|
|
96
|
+
const v = trim(s);
|
|
97
|
+
return /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}$/.test(v) ? `${v}-05:00` : v;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
async function* tenderRows(recs) {
|
|
101
|
+
for await (const get of recs) {
|
|
102
|
+
yield {
|
|
103
|
+
reference_number: trim(get('referenceNumber-numeroReference')),
|
|
104
|
+
amendment_number: trim(get('amendmentNumber-numeroModification')),
|
|
105
|
+
solicitation_number: trim(get('solicitationNumber-numeroSollicitation')),
|
|
106
|
+
title_en: cleanText(get('title-titre-eng')),
|
|
107
|
+
title_fr: cleanText(get('title-titre-fra')),
|
|
108
|
+
description_en: cleanText(get('tenderDescription-descriptionAppelOffres-eng')),
|
|
109
|
+
description_fr: cleanText(get('tenderDescription-descriptionAppelOffres-fra')),
|
|
110
|
+
publication_date: trim(get('publicationDate-datePublication')),
|
|
111
|
+
closing_date: closing(get('tenderClosingDate-appelOffresDateCloture')),
|
|
112
|
+
amendment_date: trim(get('amendmentDate-dateModification')),
|
|
113
|
+
expected_start_date: trim(get('expectedContractStartDate-dateDebutContratPrevue')),
|
|
114
|
+
expected_end_date: trim(get('expectedContractEndDate-dateFinContratPrevue')),
|
|
115
|
+
status_en: trim(get('tenderStatus-appelOffresStatut-eng')),
|
|
116
|
+
status_fr: trim(get('tenderStatus-appelOffresStatut-fra')),
|
|
117
|
+
categories: multi(get('procurementCategory-categorieApprovisionnement')),
|
|
118
|
+
notice_type_en: trim(get('noticeType-avisType-eng')),
|
|
119
|
+
notice_type_fr: trim(get('noticeType-avisType-fra')),
|
|
120
|
+
procurement_method_en: trim(get('procurementMethod-methodeApprovisionnement-eng')),
|
|
121
|
+
procurement_method_fr: trim(get('procurementMethod-methodeApprovisionnement-fra')),
|
|
122
|
+
selection_criteria_en: trim(get('selectionCriteria-criteresSelection-eng')),
|
|
123
|
+
selection_criteria_fr: trim(get('selectionCriteria-criteresSelection-fra')),
|
|
124
|
+
limited_tendering_reason_en: multi(get('limitedTenderingReason-raisonAppelOffresLimite-eng')),
|
|
125
|
+
limited_tendering_reason_fr: multi(get('limitedTenderingReason-raisonAppelOffresLimite-fra')),
|
|
126
|
+
trade_agreements_en: multi(get('tradeAgreements-accordsCommerciaux-eng')),
|
|
127
|
+
trade_agreements_fr: multi(get('tradeAgreements-accordsCommerciaux-fra')),
|
|
128
|
+
regions_of_opportunity_en: multi(get('regionsOfOpportunity-regionAppelOffres-eng')),
|
|
129
|
+
regions_of_opportunity_fr: multi(get('regionsOfOpportunity-regionAppelOffres-fra')),
|
|
130
|
+
regions_of_delivery_en: multi(get('regionsOfDelivery-regionsLivraison-eng')),
|
|
131
|
+
regions_of_delivery_fr: multi(get('regionsOfDelivery-regionsLivraison-fra')),
|
|
132
|
+
unspsc_codes: multi(get('unspsc')),
|
|
133
|
+
unspsc_en: multi(get('unspscDescription-eng')),
|
|
134
|
+
unspsc_fr: multi(get('unspscDescription-fra')),
|
|
135
|
+
gsin_codes: multi(get('gsin-nibs')),
|
|
136
|
+
gsin_en: multi(get('gsinDescription-nibsDescription-eng')),
|
|
137
|
+
gsin_fr: multi(get('gsinDescription-nibsDescription-fra')),
|
|
138
|
+
buyer_en: trim(get('contractingEntityName-nomEntitContractante-eng')),
|
|
139
|
+
buyer_fr: trim(get('contractingEntityName-nomEntitContractante-fra')),
|
|
140
|
+
buyer_address_en: address(get, 'eng'),
|
|
141
|
+
buyer_address_fr: address(get, 'fra'),
|
|
142
|
+
end_user_en: multi(get('endUserEntitiesName-nomEntitesUtilisateurFinal-eng')),
|
|
143
|
+
end_user_fr: multi(get('endUserEntitiesName-nomEntitesUtilisateurFinal-fra')),
|
|
144
|
+
contact_name: trim(get('contactInfoName-informationsContactNom')),
|
|
145
|
+
contact_email: trim(get('contactInfoEmail-informationsContactCourriel')),
|
|
146
|
+
contact_phone: trim(get('contactInfoPhone-contactInfoTelephone')),
|
|
147
|
+
notice_url_en: trim(get('noticeURL-URLavis-eng')),
|
|
148
|
+
notice_url_fr: trim(get('noticeURL-URLavis-fra')),
|
|
149
|
+
attachments_en: trim(get('attachment-piecesJointes-eng')),
|
|
150
|
+
attachments_fr: trim(get('attachment-piecesJointes-fra')),
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
// The source has one row per agreement per delivery point; collapse to one row
|
|
156
|
+
// per agreement and supplier with the delivery points merged.
|
|
157
|
+
async function* standingOfferRows(recs) {
|
|
158
|
+
const merged = new Map();
|
|
159
|
+
for await (const get of recs) {
|
|
160
|
+
const row = {
|
|
161
|
+
agreement_number: trim(get('agreement-number')),
|
|
162
|
+
agreement_type_en: trim(get('agreement-type_en')),
|
|
163
|
+
agreement_type_fr: trim(get('agreement-type_fr')),
|
|
164
|
+
title_en: cleanText(get('sosa-description_en')),
|
|
165
|
+
title_fr: cleanText(get('sosa-description_fr')),
|
|
166
|
+
commodity_codes: quotedList(get('commodity')),
|
|
167
|
+
commodity_en: quotedList(get('commodity-description_en')),
|
|
168
|
+
commodity_fr: quotedList(get('commodity-description_fr')),
|
|
169
|
+
supplier_name: trim(get('supplier-standardized-name')),
|
|
170
|
+
supplier_legal_name: trim(get('supplier-legal-name')),
|
|
171
|
+
supplier_operating_name: trim(get('supplier-operating-name')),
|
|
172
|
+
department_en: trim(get('end-user-entity_en')),
|
|
173
|
+
department_fr: trim(get('end-user-entity_fr')),
|
|
174
|
+
award_date: trim(get('award-date')),
|
|
175
|
+
expiry_date: trim(get('expiry-date')),
|
|
176
|
+
};
|
|
177
|
+
const key = JSON.stringify(row);
|
|
178
|
+
let entry = merged.get(key);
|
|
179
|
+
if (!entry) merged.set(key, (entry = { row, points: new Map() }));
|
|
180
|
+
const code = trim(get('delivery-point-code'));
|
|
181
|
+
entry.points.set(code, [trim(get('delivery-point_en')), trim(get('delivery-point_fr'))]);
|
|
182
|
+
}
|
|
183
|
+
for (const { row, points } of merged.values()) {
|
|
184
|
+
yield {
|
|
185
|
+
...row,
|
|
186
|
+
delivery_point_codes: [...points.keys()].join('\n'),
|
|
187
|
+
delivery_points_en: [...points.values()].map((p) => p[0]).filter(Boolean).join('\n'),
|
|
188
|
+
delivery_points_fr: [...points.values()].map((p) => p[1]).filter(Boolean).join('\n'),
|
|
189
|
+
};
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
// Town, province and country only: some suppliers are individuals, and their
|
|
194
|
+
// street address adds nothing to a search.
|
|
195
|
+
function supplierLocation(get, lang) {
|
|
196
|
+
return [
|
|
197
|
+
get(`supplierAddressCity-fournisseurAdresseVille-${lang}`),
|
|
198
|
+
get(`supplierAddressProvince-fournisseurAdresseProvince-${lang}`),
|
|
199
|
+
get(`supplierAddressCountry-fournisseurAdressePays-${lang}`),
|
|
200
|
+
].map(trim).filter(Boolean).join(', ');
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
// One row per notice, carrying its latest amendment. The total contract value
|
|
204
|
+
// is cumulative across amendments but some buyers leave it at zero and fill
|
|
205
|
+
// only the amount, so the first non-zero of the two is kept. Standing offers
|
|
206
|
+
// and supply arrangements have no value and stay at zero.
|
|
207
|
+
async function* awardRows(recs) {
|
|
208
|
+
for await (const get of recs) {
|
|
209
|
+
const total = Number(get('totalContractValue-valeurTotaleContrat'));
|
|
210
|
+
const amount = Number(get('contractAmount-montantContrat'));
|
|
211
|
+
yield {
|
|
212
|
+
reference_number: trim(get('referenceNumber-numeroReference')),
|
|
213
|
+
amendment_number: trim(get('amendmentNumber-numeroModification')),
|
|
214
|
+
solicitation_number: trim(get('solicitationNumber-numeroSollicitation')),
|
|
215
|
+
contract_number: trim(get('contractNumber-numeroContrat')),
|
|
216
|
+
title_en: cleanText(get('title-titre-eng')),
|
|
217
|
+
title_fr: cleanText(get('title-titre-fra')),
|
|
218
|
+
description_en: cleanText(get('awardDescription-descriptionAttribution-eng')),
|
|
219
|
+
description_fr: cleanText(get('awardDescription-descriptionAttribution-fra')),
|
|
220
|
+
publication_date: trim(get('publicationDate-datePublication')),
|
|
221
|
+
award_date: trim(get('contractAwardDate-dateAttributionContrat')),
|
|
222
|
+
amendment_date: trim(get('amendmentDate-dateModification')),
|
|
223
|
+
contract_start_date: trim(get('contractStartDate-contratDateDebut')),
|
|
224
|
+
contract_end_date: trim(get('contractEndDate-dateFinContrat')),
|
|
225
|
+
value: total > 0 ? total : amount > 0 ? amount : 0,
|
|
226
|
+
currency: trim(get('contractCurrency-contratMonnaie')),
|
|
227
|
+
status_en: trim(get('awardStatus-attributionStatut-eng')),
|
|
228
|
+
status_fr: trim(get('awardStatus-attributionStatut-fra')),
|
|
229
|
+
instrument_type_en: trim(get('instrumentType-typeInstrument-eng')),
|
|
230
|
+
instrument_type_fr: trim(get('instrumentType-typeInstrument-fra')),
|
|
231
|
+
amendment_type_en: trim(get('amendmentType-typeModification-eng')),
|
|
232
|
+
amendment_type_fr: trim(get('amendmentType-typeModification-fra')),
|
|
233
|
+
categories: multi(get('procurementCategory-categorieApprovisionnement')),
|
|
234
|
+
notice_type_en: trim(get('noticeType-avisType-eng')),
|
|
235
|
+
notice_type_fr: trim(get('noticeType-avisType-fra')),
|
|
236
|
+
procurement_method_en: trim(get('procurementMethod-methodeApprovisionnement-eng')),
|
|
237
|
+
procurement_method_fr: trim(get('procurementMethod-methodeApprovisionnement-fra')),
|
|
238
|
+
selection_criteria_en: trim(get('selectionCriteria-criteresSelection-eng')),
|
|
239
|
+
selection_criteria_fr: trim(get('selectionCriteria-criteresSelection-fra')),
|
|
240
|
+
limited_tendering_reason_en: multi(get('limitedTenderingReason-raisonAppelOffresLimite-eng')),
|
|
241
|
+
limited_tendering_reason_fr: multi(get('limitedTenderingReason-raisonAppelOffresLimite-fra')),
|
|
242
|
+
trade_agreements_en: multi(get('tradeAgreements-accordsCommerciaux-eng')),
|
|
243
|
+
trade_agreements_fr: multi(get('tradeAgreements-accordsCommerciaux-fra')),
|
|
244
|
+
regions_of_delivery_en: multi(get('regionsOfDelivery-regionsLivraison-eng')),
|
|
245
|
+
regions_of_delivery_fr: multi(get('regionsOfDelivery-regionsLivraison-fra')),
|
|
246
|
+
unspsc_codes: multi(get('unspsc')),
|
|
247
|
+
unspsc_en: multi(get('unspscDescription-eng')),
|
|
248
|
+
unspsc_fr: multi(get('unspscDescription-fra')),
|
|
249
|
+
supplier_name:
|
|
250
|
+
trim(get('supplierLegalName-nomLegalFournisseur-eng')) || trim(get('supplierLegalName-nomLegalFournisseur-fra')),
|
|
251
|
+
supplier_location_en: supplierLocation(get, 'eng'),
|
|
252
|
+
supplier_location_fr: supplierLocation(get, 'fra'),
|
|
253
|
+
buyer_en: trim(get('contractingEntityName-nomEntitContractante-eng')),
|
|
254
|
+
buyer_fr: trim(get('contractingEntityName-nomEntitContractante-fra')),
|
|
255
|
+
end_user_en: multi(get('endUserEntitiesName-nomEntitesUtilisateurFinal-eng')),
|
|
256
|
+
end_user_fr: multi(get('endUserEntitiesName-nomEntitesUtilisateurFinal-fra')),
|
|
257
|
+
contact_name: trim(get('contactInfoName-informationsContactNom')),
|
|
258
|
+
contact_email: trim(get('contactInfoEmail-informationsContactCourriel')),
|
|
259
|
+
contact_phone: trim(get('contactInfoPhone-contactInfoTelephone')),
|
|
260
|
+
};
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
// Award notices come in one file per federal fiscal year (April to March),
|
|
265
|
+
// each holding the notices last published in that year. The first CanadaBuys
|
|
266
|
+
// year is 2022-2023. Only the current year's file changes from day to day; it
|
|
267
|
+
// may not exist yet in the first days of April, so it is not required.
|
|
268
|
+
const FIRST_AWARD_YEAR = 2022;
|
|
269
|
+
|
|
270
|
+
function awardFiles(now) {
|
|
271
|
+
const local = new Date(now - 5 * HOUR);
|
|
272
|
+
const current = local.getUTCFullYear() - (local.getUTCMonth() < 3 ? 1 : 0);
|
|
273
|
+
const files = [];
|
|
274
|
+
for (let year = FIRST_AWARD_YEAR; year <= current; year++) {
|
|
275
|
+
files.push({
|
|
276
|
+
id: `awards_${year}`,
|
|
277
|
+
label: `Award notices, ${year}-${year + 1}`,
|
|
278
|
+
url: `${PUB}${year}-${year + 1}-awardNotice-avisAttribution.csv`,
|
|
279
|
+
table: 'awards',
|
|
280
|
+
rows: awardRows,
|
|
281
|
+
onConflict: 'REPLACE',
|
|
282
|
+
ttl: (year === current ? 6 : 24) * HOUR,
|
|
283
|
+
required: year < current,
|
|
284
|
+
});
|
|
285
|
+
}
|
|
286
|
+
return files;
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
// `ttl` is how long a successful check is trusted before the next conditional
|
|
290
|
+
// request. `onConflict` decides which file wins when both carry a notice.
|
|
291
|
+
// A `supplementary` file does not count towards the data date reported for
|
|
292
|
+
// its table. The list is fixed when the server starts.
|
|
293
|
+
export const FILES = [
|
|
294
|
+
{
|
|
295
|
+
id: 'open_tenders',
|
|
296
|
+
label: 'Open tender notices',
|
|
297
|
+
url: `${PUB}openTenderNotice-ouvertAvisAppelOffres.csv`,
|
|
298
|
+
table: 'tenders',
|
|
299
|
+
rows: tenderRows,
|
|
300
|
+
onConflict: 'REPLACE',
|
|
301
|
+
ttl: 6 * HOUR,
|
|
302
|
+
required: true,
|
|
303
|
+
},
|
|
304
|
+
{
|
|
305
|
+
// Published every two hours; may be empty early in the day and may hold
|
|
306
|
+
// notices the daily open file has not picked up yet.
|
|
307
|
+
id: 'new_tenders',
|
|
308
|
+
label: 'New tender notices',
|
|
309
|
+
url: `${PUB}newTenderNotice-nouvelAvisAppelOffres.csv`,
|
|
310
|
+
table: 'tenders',
|
|
311
|
+
rows: tenderRows,
|
|
312
|
+
onConflict: 'IGNORE',
|
|
313
|
+
ttl: 2 * HOUR,
|
|
314
|
+
required: false,
|
|
315
|
+
supplementary: true,
|
|
316
|
+
},
|
|
317
|
+
{
|
|
318
|
+
id: 'standing_offers',
|
|
319
|
+
label: 'Active standing offers and supply arrangements',
|
|
320
|
+
url: 'https://sosa.canadabuys.canada.ca/cds/opendata/tpsgc-pwgsc_ocama-sosa.csv',
|
|
321
|
+
table: 'standing_offers',
|
|
322
|
+
rows: standingOfferRows,
|
|
323
|
+
onConflict: 'REPLACE',
|
|
324
|
+
ttl: 24 * HOUR,
|
|
325
|
+
required: true,
|
|
326
|
+
},
|
|
327
|
+
...awardFiles(Date.now()),
|
|
328
|
+
];
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
// HTTPS GET with a streamed body. Some government hosts serve their
|
|
2
|
+
// certificate without the intermediate that signed it, which browsers repair
|
|
3
|
+
// by fetching the issuer named in the certificate (AIA). Node does not, so
|
|
4
|
+
// the same repair is done here, accepting the issuer only when it is signed
|
|
5
|
+
// by one of Node's bundled root authorities.
|
|
6
|
+
|
|
7
|
+
import { X509Certificate } from 'node:crypto';
|
|
8
|
+
import http from 'node:http';
|
|
9
|
+
import https from 'node:https';
|
|
10
|
+
import tls from 'node:tls';
|
|
11
|
+
|
|
12
|
+
const TIMEOUT_MS = 120_000;
|
|
13
|
+
const MAX_CERT_BYTES = 64 * 1024;
|
|
14
|
+
/** @type {Map<string, string>} host -> verified issuer certificate (PEM) */
|
|
15
|
+
const issuers = new Map();
|
|
16
|
+
|
|
17
|
+
function get(url, headers, ca) {
|
|
18
|
+
return new Promise((resolve, reject) => {
|
|
19
|
+
https.get(url, { headers, ca, signal: AbortSignal.timeout(TIMEOUT_MS) }, resolve).on('error', reject);
|
|
20
|
+
});
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function peerCertificate(host) {
|
|
24
|
+
return new Promise((resolve, reject) => {
|
|
25
|
+
const socket = tls.connect({ host, port: 443, servername: host, rejectUnauthorized: false }, () => {
|
|
26
|
+
const cert = socket.getPeerX509Certificate();
|
|
27
|
+
socket.destroy();
|
|
28
|
+
cert ? resolve(cert) : reject(new Error('the server sent no certificate'));
|
|
29
|
+
});
|
|
30
|
+
socket.setTimeout(TIMEOUT_MS, () => socket.destroy(new Error('timed out reading the server certificate')));
|
|
31
|
+
socket.on('error', reject);
|
|
32
|
+
});
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function fetchCertificate(url) {
|
|
36
|
+
return new Promise((resolve, reject) => {
|
|
37
|
+
http
|
|
38
|
+
.get(url, { signal: AbortSignal.timeout(TIMEOUT_MS) }, (res) => {
|
|
39
|
+
if (res.statusCode !== 200) {
|
|
40
|
+
res.resume();
|
|
41
|
+
reject(new Error(`issuer certificate request returned HTTP ${res.statusCode}`));
|
|
42
|
+
return;
|
|
43
|
+
}
|
|
44
|
+
const chunks = [];
|
|
45
|
+
let size = 0;
|
|
46
|
+
res.on('data', (chunk) => {
|
|
47
|
+
size += chunk.length;
|
|
48
|
+
if (size > MAX_CERT_BYTES) res.destroy(new Error('issuer certificate is too large'));
|
|
49
|
+
else chunks.push(chunk);
|
|
50
|
+
});
|
|
51
|
+
res.on('end', () => resolve(new X509Certificate(Buffer.concat(chunks))));
|
|
52
|
+
res.on('error', reject);
|
|
53
|
+
})
|
|
54
|
+
.on('error', reject);
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
async function missingIssuer(host) {
|
|
59
|
+
const leaf = await peerCertificate(host);
|
|
60
|
+
const url = /CA Issuers - URI:(http:\/\/\S+)/.exec(leaf.infoAccess ?? '')?.[1];
|
|
61
|
+
if (!url) throw new Error('the server certificate does not name its issuer');
|
|
62
|
+
const issuer = await fetchCertificate(url);
|
|
63
|
+
const now = Date.now();
|
|
64
|
+
const current = Date.parse(issuer.validFrom) <= now && now <= Date.parse(issuer.validTo);
|
|
65
|
+
const signsLeaf = leaf.checkIssued(issuer) && leaf.verify(issuer.publicKey);
|
|
66
|
+
const trusted = tls.rootCertificates.some((pem) => {
|
|
67
|
+
const root = new X509Certificate(pem);
|
|
68
|
+
return issuer.checkIssued(root) && issuer.verify(root.publicKey);
|
|
69
|
+
});
|
|
70
|
+
if (!current || !signsLeaf || !trusted) throw new Error('the server certificate chain could not be verified');
|
|
71
|
+
return issuer.toString();
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* @param {string} url https URL
|
|
76
|
+
* @param {Record<string, string>} headers
|
|
77
|
+
* @returns {Promise<import('node:http').IncomingMessage>} response with an unread body
|
|
78
|
+
*/
|
|
79
|
+
export async function download(url, headers) {
|
|
80
|
+
const { host } = new URL(url);
|
|
81
|
+
const withIssuer = (pem) => [...tls.rootCertificates, pem];
|
|
82
|
+
if (issuers.has(host)) return get(url, headers, withIssuer(issuers.get(host)));
|
|
83
|
+
try {
|
|
84
|
+
return await get(url, headers);
|
|
85
|
+
} catch (err) {
|
|
86
|
+
if (err.code !== 'UNABLE_TO_VERIFY_LEAF_SIGNATURE') throw err;
|
|
87
|
+
const pem = await missingIssuer(host);
|
|
88
|
+
issuers.set(host, pem);
|
|
89
|
+
return get(url, headers, withIssuer(pem));
|
|
90
|
+
}
|
|
91
|
+
}
|
package/server/index.js
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Entry point: `node server/index.js` speaks MCP on stdin/stdout.
|
|
3
|
+
|
|
4
|
+
// Probe before loading anything else so an unsuitable Node gets one clear
|
|
5
|
+
// message. Node 22.16+ and 24+ bundle SQLite with full-text search; older
|
|
6
|
+
// releases and the 23.x line do not.
|
|
7
|
+
try {
|
|
8
|
+
const { DatabaseSync } = await import('node:sqlite');
|
|
9
|
+
const probe = new DatabaseSync(':memory:');
|
|
10
|
+
probe.exec('CREATE VIRTUAL TABLE probe USING fts5(text)');
|
|
11
|
+
probe.close();
|
|
12
|
+
} catch {
|
|
13
|
+
console.error(`northbid needs Node.js 22.16 or newer (24 or newer recommended); this is ${process.version}.`);
|
|
14
|
+
process.exit(1);
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
const { readFileSync } = await import('node:fs');
|
|
18
|
+
const { serve } = await import('./mcp.js');
|
|
19
|
+
const { defaultDataDir, Store } = await import('./store.js');
|
|
20
|
+
const { createTools, INSTRUCTIONS, PROMPTS } = await import('./tools.js');
|
|
21
|
+
|
|
22
|
+
const pkg = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8'));
|
|
23
|
+
const store = new Store(defaultDataDir(), {
|
|
24
|
+
offline: process.env.NORTHBID_OFFLINE === '1',
|
|
25
|
+
// The CanadaBuys firewall rejects Node's default agent and any agent containing a URL.
|
|
26
|
+
userAgent: `northbid/${pkg.version}`,
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
await serve({
|
|
30
|
+
info: { name: 'northbid', title: 'Northbid: Canadian procurement search', version: pkg.version },
|
|
31
|
+
instructions: INSTRUCTIONS,
|
|
32
|
+
tools: createTools(store),
|
|
33
|
+
prompts: PROMPTS,
|
|
34
|
+
});
|
|
35
|
+
store.close();
|
package/server/mcp.js
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
// Minimal Model Context Protocol server over stdio: newline-delimited JSON-RPC
|
|
2
|
+
// 2.0 with the tools and prompts capabilities.
|
|
3
|
+
|
|
4
|
+
import { createInterface } from 'node:readline';
|
|
5
|
+
import { InputError } from './search.js';
|
|
6
|
+
import { validate } from './validate.js';
|
|
7
|
+
|
|
8
|
+
const PROTOCOL_VERSIONS = ['2025-11-25', '2025-06-18', '2025-03-26', '2024-11-05'];
|
|
9
|
+
|
|
10
|
+
class RpcError extends Error {
|
|
11
|
+
constructor(code, message) {
|
|
12
|
+
super(message);
|
|
13
|
+
this.code = code;
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* @param {object} server
|
|
19
|
+
* @param {{ name: string, title: string, version: string }} server.info
|
|
20
|
+
* @param {string} server.instructions
|
|
21
|
+
* @param {Array<{ name: string, handler: (args: any) => Promise<object>, inputSchema: any }>} server.tools
|
|
22
|
+
* @param {Array<{ name: string, render: (args: any) => string, arguments: any[] }>} server.prompts
|
|
23
|
+
* @param {NodeJS.ReadableStream} [input]
|
|
24
|
+
* @param {NodeJS.WritableStream} [output]
|
|
25
|
+
* @returns {Promise<void>} resolves when the input stream ends
|
|
26
|
+
*/
|
|
27
|
+
export function serve({ info, instructions, tools, prompts }, input = process.stdin, output = process.stdout) {
|
|
28
|
+
const send = (message) => output.write(`${JSON.stringify(message)}\n`);
|
|
29
|
+
const toolList = tools.map(({ handler, ...definition }) => definition);
|
|
30
|
+
const promptList = prompts.map(({ render, ...definition }) => definition);
|
|
31
|
+
|
|
32
|
+
const methods = {
|
|
33
|
+
initialize: (params) => ({
|
|
34
|
+
protocolVersion: PROTOCOL_VERSIONS.includes(params?.protocolVersion) ? params.protocolVersion : PROTOCOL_VERSIONS[0],
|
|
35
|
+
capabilities: { tools: {}, prompts: {} },
|
|
36
|
+
serverInfo: info,
|
|
37
|
+
instructions,
|
|
38
|
+
}),
|
|
39
|
+
ping: () => ({}),
|
|
40
|
+
'tools/list': () => ({ tools: toolList }),
|
|
41
|
+
'tools/call': async (params) => {
|
|
42
|
+
const tool = tools.find((t) => t.name === params?.name);
|
|
43
|
+
if (!tool) throw new RpcError(-32602, `Unknown tool: ${params?.name}`);
|
|
44
|
+
try {
|
|
45
|
+
const result = await tool.handler(validate(tool.inputSchema, params.arguments));
|
|
46
|
+
return { content: [{ type: 'text', text: JSON.stringify(result) }] };
|
|
47
|
+
} catch (err) {
|
|
48
|
+
if (!(err instanceof InputError)) console.error(`[northbid] ${tool.name} failed:`, err);
|
|
49
|
+
return { content: [{ type: 'text', text: err.message }], isError: true };
|
|
50
|
+
}
|
|
51
|
+
},
|
|
52
|
+
'prompts/list': () => ({ prompts: promptList }),
|
|
53
|
+
'prompts/get': (params) => {
|
|
54
|
+
const prompt = prompts.find((p) => p.name === params?.name);
|
|
55
|
+
if (!prompt) throw new RpcError(-32602, `Unknown prompt: ${params?.name}`);
|
|
56
|
+
const args = params.arguments ?? {};
|
|
57
|
+
for (const a of prompt.arguments) {
|
|
58
|
+
if (a.required && !args[a.name]) throw new RpcError(-32602, `Missing prompt argument: ${a.name}`);
|
|
59
|
+
}
|
|
60
|
+
return { messages: [{ role: 'user', content: { type: 'text', text: prompt.render(args) } }] };
|
|
61
|
+
},
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
async function handle(line) {
|
|
65
|
+
let message;
|
|
66
|
+
try {
|
|
67
|
+
message = JSON.parse(line);
|
|
68
|
+
} catch {
|
|
69
|
+
send({ jsonrpc: '2.0', id: null, error: { code: -32700, message: 'Parse error' } });
|
|
70
|
+
return;
|
|
71
|
+
}
|
|
72
|
+
const isRequest = message.id !== undefined && message.id !== null && typeof message.method === 'string';
|
|
73
|
+
if (!isRequest) return; // notifications and responses need no reply
|
|
74
|
+
const method = methods[message.method];
|
|
75
|
+
if (!method) {
|
|
76
|
+
send({ jsonrpc: '2.0', id: message.id, error: { code: -32601, message: `Method not found: ${message.method}` } });
|
|
77
|
+
return;
|
|
78
|
+
}
|
|
79
|
+
try {
|
|
80
|
+
send({ jsonrpc: '2.0', id: message.id, result: await method(message.params) });
|
|
81
|
+
} catch (err) {
|
|
82
|
+
const code = err instanceof RpcError ? err.code : -32603;
|
|
83
|
+
if (code === -32603) console.error('[northbid] internal error:', err);
|
|
84
|
+
send({ jsonrpc: '2.0', id: message.id, error: { code, message: err.message } });
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
return new Promise((resolve) => {
|
|
89
|
+
const pending = new Set();
|
|
90
|
+
const rl = createInterface({ input, crlfDelay: Infinity });
|
|
91
|
+
rl.on('line', (line) => {
|
|
92
|
+
if (!line.trim()) return;
|
|
93
|
+
const job = handle(line).finally(() => pending.delete(job));
|
|
94
|
+
pending.add(job);
|
|
95
|
+
});
|
|
96
|
+
rl.on('close', () => Promise.all(pending).then(() => resolve()));
|
|
97
|
+
});
|
|
98
|
+
}
|