@idea1/filekit 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +72 -0
- package/filekit.js +498 -0
- package/package.json +25 -0
package/README.md
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# @datafabriq/filekit
|
|
2
|
+
|
|
3
|
+
A reader and writer for the spreadsheets agent-driven data pipelines depend on — CSV and
|
|
4
|
+
XLSX files a client uploads that a downstream process has to parse correctly every time.
|
|
5
|
+
Pure Node (`fs` + `zlib`), no dependencies, no build step.
|
|
6
|
+
|
|
7
|
+
```js
|
|
8
|
+
const fk = require('@datafabriq/filekit');
|
|
9
|
+
|
|
10
|
+
fk.parseCsv(text, { headerRow }) // quoted fields, comma-thousands; headerRow skips a preamble
|
|
11
|
+
fk.readXlsx(pathOrBuffer) // reads .xlsx directly — the unzip is built in
|
|
12
|
+
fk.writeXlsx(path, sheets) // one tab per dataset; row 0 is the header
|
|
13
|
+
|
|
14
|
+
fk.num(v) // '$1,234.50' → 1234.5; '(312,571.01)' → -312571.01
|
|
15
|
+
fk.excelSerialToYM(n) // 46143 → '2026-05'
|
|
16
|
+
fk.findMonthColumns(header) // discover a MON YYYY matrix by header, never by index
|
|
17
|
+
fk.findColumnBySerialMonth(sheet, headerRow, '2026-05')
|
|
18
|
+
fk.findRow(sheet, (row, n) => …) // locate a block by its label
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
`readXlsx` returns real sheet names in workbook order; address a sheet by name
|
|
22
|
+
(`wb.sheet('GL 24500 Accrued Bonus')`) rather than by index. Cells are 1-based and
|
|
23
|
+
Excel-shaped: `sheet.at(12, 3)` is cell `C12`.
|
|
24
|
+
|
|
25
|
+
## Reading messy source files
|
|
26
|
+
|
|
27
|
+
Two properties of these files break naive parsers, and both are covered by the tests:
|
|
28
|
+
|
|
29
|
+
- **Month columns move.** A commission ledger's `MON YYYY` matrix grows a column each
|
|
30
|
+
period, and a bonus workbook's month headers are Excel serial dates. Locate the close
|
|
31
|
+
month by matching the header; a fixed column index silently reads the wrong month.
|
|
32
|
+
- **A cell's value is not always its text.** Totals are formula cells whose value lives in
|
|
33
|
+
`<v>`; empty cells are self-closing and carry no value at all; text may be a shared
|
|
34
|
+
string or an inline string. `filekit` resolves all four.
|
|
35
|
+
|
|
36
|
+
Locate a block by the labels around it (`Beginning`, `New Accruals`, `Payments`, `Ending`,
|
|
37
|
+
the GL numbers, etc.) rather than by row number, so an inserted row does not shift the read.
|
|
38
|
+
|
|
39
|
+
## Writing a reference workbook
|
|
40
|
+
|
|
41
|
+
`writeXlsx` produces workbooks a Custom Files connector can ingest — one tab per dataset,
|
|
42
|
+
`snake_case` headers on row 1, and conventions bronze expects:
|
|
43
|
+
|
|
44
|
+
- Dates as literal `YYYY-MM-DD` text, never Excel serials, so `TRY_CAST(… AS DATE)` holds.
|
|
45
|
+
- An omitted cell for a null; an empty cell ingests as `NULL`, `0` does not.
|
|
46
|
+
- Values written as numbers, text as inline strings, XML-escaped.
|
|
47
|
+
- Every column ingests as `STRING`; the reading spec does the `TRY_CAST`.
|
|
48
|
+
|
|
49
|
+
## Tests
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
npm test
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
Self-contained — every fixture is built in-process, so the suite carries no customer data
|
|
56
|
+
and runs anywhere. It asserts the cell shapes above: formula cells, self-closing cells,
|
|
57
|
+
shared strings with entities, serial dates, a header below a nine-row preamble, and a
|
|
58
|
+
write→read round-trip preserving nulls, negatives, and escaping.
|
|
59
|
+
|
|
60
|
+
## Origin
|
|
61
|
+
|
|
62
|
+
Extracted from Concept3D's month-end close kit (`datafabriq-ops` → `context/revenue-reporting/close-tools/`),
|
|
63
|
+
where the same reader already served two teams' file shapes (Concept3D's QuotaPath/bonus
|
|
64
|
+
files, Intake's Amazon/Ed's-workbook files) before this package existed.
|
|
65
|
+
|
|
66
|
+
## Publishing
|
|
67
|
+
|
|
68
|
+
Published as `@idea1/filekit` (public), same npm org as `@idea1/cli`. Pushing a
|
|
69
|
+
`filekit-v*.*.*` tag runs `.github/workflows/publish-filekit.yml`, which runs `npm run build`
|
|
70
|
+
(the test suite) then `npm run deploy` (`npm publish --access public`), authenticated via
|
|
71
|
+
the `NPM_TOKEN` repo secret — an npm Automation token for a member of the `idea1` npm org.
|
|
72
|
+
Bump the version in `package.json` first, since npm rejects re-publishing an existing version.
|
package/filekit.js
ADDED
|
@@ -0,0 +1,498 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
/*
|
|
3
|
+
* filekit — messy-file reader toolkit for the close kit.
|
|
4
|
+
*
|
|
5
|
+
* Reads the uploaded spreadsheets a month-end close depends on: CSVs with a
|
|
6
|
+
* preamble before the real header, and .xlsx workbooks (read directly — the
|
|
7
|
+
* unzip is built in, no npm dependency, no pre-unzipping).
|
|
8
|
+
*
|
|
9
|
+
* Pure Node (fs + zlib). No external packages. Same reader serves every team:
|
|
10
|
+
* Concept3D — QuotaPath commission CSV (MON YYYY matrix), accrued-bonus XLSX.
|
|
11
|
+
* Intake — Amazon Main report CSV (header on row 10), Ed's XLSX.
|
|
12
|
+
*
|
|
13
|
+
* Exports: parseCsv, readXlsx, num, excelSerialToDate, excelSerialToYM,
|
|
14
|
+
* findMonthColumns, findRow, unzip.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
const fs = require('fs');
|
|
18
|
+
const zlib = require('zlib');
|
|
19
|
+
|
|
20
|
+
/* ------------------------------------------------------------------ *
|
|
21
|
+
* Numbers & dates
|
|
22
|
+
* ------------------------------------------------------------------ */
|
|
23
|
+
|
|
24
|
+
/** Coerce a spreadsheet cell to a number, tolerating $ , % whitespace and (parens) negatives. */
|
|
25
|
+
function num(v) {
|
|
26
|
+
if (v == null || v === '') return 0;
|
|
27
|
+
let s = String(v).trim();
|
|
28
|
+
let neg = false;
|
|
29
|
+
if (/^\(.*\)$/.test(s)) { neg = true; s = s.slice(1, -1); }
|
|
30
|
+
s = s.replace(/[$,%\s]/g, '');
|
|
31
|
+
const x = parseFloat(s);
|
|
32
|
+
if (isNaN(x)) return 0;
|
|
33
|
+
return neg ? -x : x;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** Excel serial date (1900 system) → JS Date (UTC). */
|
|
37
|
+
function excelSerialToDate(n) {
|
|
38
|
+
return new Date(Date.UTC(1899, 11, 30) + Math.round(Number(n)) * 86400000);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** Excel serial date → "YYYY-MM". */
|
|
42
|
+
function excelSerialToYM(n) {
|
|
43
|
+
const d = excelSerialToDate(n);
|
|
44
|
+
return d.getUTCFullYear() + '-' + String(d.getUTCMonth() + 1).padStart(2, '0');
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/* ------------------------------------------------------------------ *
|
|
48
|
+
* CSV
|
|
49
|
+
* ------------------------------------------------------------------ */
|
|
50
|
+
|
|
51
|
+
/** RFC-4180-ish tokenizer: handles quoted fields, "" escapes, CRLF. → string[][] */
|
|
52
|
+
function tokenizeCsv(text) {
|
|
53
|
+
const rows = [];
|
|
54
|
+
let row = [], field = '', inQ = false;
|
|
55
|
+
for (let i = 0; i < text.length; i++) {
|
|
56
|
+
const ch = text[i];
|
|
57
|
+
if (inQ) {
|
|
58
|
+
if (ch === '"') { if (text[i + 1] === '"') { field += '"'; i++; } else inQ = false; }
|
|
59
|
+
else field += ch;
|
|
60
|
+
} else {
|
|
61
|
+
if (ch === '"') inQ = true;
|
|
62
|
+
else if (ch === ',') { row.push(field); field = ''; }
|
|
63
|
+
else if (ch === '\n') { row.push(field); rows.push(row); row = []; field = ''; }
|
|
64
|
+
else if (ch === '\r') { /* skip */ }
|
|
65
|
+
else field += ch;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
if (field.length || row.length) { row.push(field); rows.push(row); }
|
|
69
|
+
return rows;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Parse CSV text into header + row objects.
|
|
74
|
+
* @param {string} text
|
|
75
|
+
* @param {object} [opts]
|
|
76
|
+
* @param {number} [opts.headerRow=0] 0-based index of the header line. Set to skip a
|
|
77
|
+
* preamble (e.g. 9 for Amazon's row-10 header).
|
|
78
|
+
* @returns {{header:string[], rows:Array<Object>, raw:string[][]}}
|
|
79
|
+
* Each row is keyed by header name; `row._cells` keeps positional access.
|
|
80
|
+
*/
|
|
81
|
+
function parseCsv(text, opts = {}) {
|
|
82
|
+
const headerRow = opts.headerRow ?? 0;
|
|
83
|
+
const raw = tokenizeCsv(text);
|
|
84
|
+
const header = (raw[headerRow] || []).map(h => h.trim());
|
|
85
|
+
const rows = [];
|
|
86
|
+
for (let r = headerRow + 1; r < raw.length; r++) {
|
|
87
|
+
const cells = raw[r];
|
|
88
|
+
if (!cells || (cells.length === 1 && cells[0] === '')) continue; // blank line
|
|
89
|
+
const o = { _cells: cells };
|
|
90
|
+
for (let c = 0; c < header.length; c++) o[header[c]] = cells[c];
|
|
91
|
+
rows.push(o);
|
|
92
|
+
}
|
|
93
|
+
return { header, rows, raw };
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Find the `MON YYYY` matrix columns in a CSV header (e.g. QuotaPath's amortization
|
|
98
|
+
* schedule). Never rely on a fixed index — the matrix grows a column each month.
|
|
99
|
+
* @returns {Array<{name:string, index:number}>} in header order.
|
|
100
|
+
*/
|
|
101
|
+
function findMonthColumns(header) {
|
|
102
|
+
const re = /^[A-Za-z]{3}\s+\d{4}$/;
|
|
103
|
+
return header
|
|
104
|
+
.map((h, i) => ({ name: (h || '').trim(), index: i }))
|
|
105
|
+
.filter(c => re.test(c.name));
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/* ------------------------------------------------------------------ *
|
|
109
|
+
* ZIP (minimal, dependency-free) — enough to open an .xlsx
|
|
110
|
+
* ------------------------------------------------------------------ */
|
|
111
|
+
|
|
112
|
+
/** Unzip a Buffer via its central directory. → Map<name, Buffer>. Handles stored + deflate. */
|
|
113
|
+
function unzip(buf) {
|
|
114
|
+
const files = new Map();
|
|
115
|
+
// Locate End Of Central Directory (scan back from the end; max 64k comment).
|
|
116
|
+
let eocd = -1;
|
|
117
|
+
const minEnd = Math.max(0, buf.length - 22 - 0xffff);
|
|
118
|
+
for (let i = buf.length - 22; i >= minEnd; i--) {
|
|
119
|
+
if (buf.readUInt32LE(i) === 0x06054b50) { eocd = i; break; }
|
|
120
|
+
}
|
|
121
|
+
if (eocd < 0) throw new Error('filekit.unzip: not a zip (no EOCD signature)');
|
|
122
|
+
const count = buf.readUInt16LE(eocd + 10);
|
|
123
|
+
let p = buf.readUInt32LE(eocd + 16); // central directory offset
|
|
124
|
+
for (let n = 0; n < count; n++) {
|
|
125
|
+
if (buf.readUInt32LE(p) !== 0x02014b50) break; // central file header
|
|
126
|
+
const method = buf.readUInt16LE(p + 10);
|
|
127
|
+
const compSize = buf.readUInt32LE(p + 20);
|
|
128
|
+
const nameLen = buf.readUInt16LE(p + 28);
|
|
129
|
+
const extraLen = buf.readUInt16LE(p + 30);
|
|
130
|
+
const cmtLen = buf.readUInt16LE(p + 32);
|
|
131
|
+
const localOff = buf.readUInt32LE(p + 42);
|
|
132
|
+
const name = buf.toString('utf8', p + 46, p + 46 + nameLen);
|
|
133
|
+
// Jump to the local header to compute where the data actually starts.
|
|
134
|
+
const lNameLen = buf.readUInt16LE(localOff + 26);
|
|
135
|
+
const lExtraLen = buf.readUInt16LE(localOff + 28);
|
|
136
|
+
const dataStart = localOff + 30 + lNameLen + lExtraLen;
|
|
137
|
+
const comp = buf.subarray(dataStart, dataStart + compSize);
|
|
138
|
+
let content;
|
|
139
|
+
if (method === 0) content = Buffer.from(comp);
|
|
140
|
+
else if (method === 8) content = zlib.inflateRawSync(comp);
|
|
141
|
+
else throw new Error('filekit.unzip: unsupported compression method ' + method + ' for ' + name);
|
|
142
|
+
files.set(name, content);
|
|
143
|
+
p += 46 + nameLen + extraLen + cmtLen;
|
|
144
|
+
}
|
|
145
|
+
return files;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/* ------------------------------------------------------------------ *
|
|
149
|
+
* XLSX
|
|
150
|
+
* ------------------------------------------------------------------ */
|
|
151
|
+
|
|
152
|
+
function colToNum(letters) {
|
|
153
|
+
let n = 0;
|
|
154
|
+
for (const ch of letters) n = n * 26 + (ch.charCodeAt(0) - 64);
|
|
155
|
+
return n;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
const XML_ENT = { '&': '&', '<': '<', '>': '>', '"': '"', ''': "'", ' ': ' ', ''': "'" };
|
|
159
|
+
function unescapeXml(s) {
|
|
160
|
+
return s.replace(/&|<|>|"|'| |'/g, m => XML_ENT[m]);
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
function parseSharedStrings(xml) {
|
|
164
|
+
const out = [];
|
|
165
|
+
let m; const re = /<si>([\s\S]*?)<\/si>/g;
|
|
166
|
+
while ((m = re.exec(xml))) {
|
|
167
|
+
const parts = [...m[1].matchAll(/<t[^>]*>([\s\S]*?)<\/t>/g)].map(x => x[1]);
|
|
168
|
+
out.push(unescapeXml(parts.join('')));
|
|
169
|
+
}
|
|
170
|
+
return out;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/** One worksheet: sparse cell grid with a 1-based Excel-style accessor. */
|
|
174
|
+
function makeSheet(xml, shared) {
|
|
175
|
+
const cells = Object.create(null); // cells[row][col] = value (1-based)
|
|
176
|
+
let maxRow = 0, maxCol = 0;
|
|
177
|
+
let rm; const rowRe = /<row[^>]*\br="(\d+)"[^>]*>([\s\S]*?)<\/row>/g;
|
|
178
|
+
while ((rm = rowRe.exec(xml))) {
|
|
179
|
+
const rn = +rm[1];
|
|
180
|
+
const line = (cells[rn] = cells[rn] || Object.create(null));
|
|
181
|
+
let cm; const cellRe = /<c r="([A-Z]+)(\d+)"([^>]*?)(?:\/>|>([\s\S]*?)<\/c>)/g;
|
|
182
|
+
while ((cm = cellRe.exec(rm[2]))) {
|
|
183
|
+
const col = colToNum(cm[1]);
|
|
184
|
+
const attrs = cm[3] || '';
|
|
185
|
+
const inner = cm[4] || '';
|
|
186
|
+
const t = (attrs.match(/t="([^"]*)"/) || [])[1];
|
|
187
|
+
let val = '';
|
|
188
|
+
const isMatch = inner.match(/<is>[\s\S]*?<t[^>]*>([\s\S]*?)<\/t>[\s\S]*?<\/is>/);
|
|
189
|
+
if (isMatch) val = unescapeXml(isMatch[1]); // inline string
|
|
190
|
+
else {
|
|
191
|
+
const v = (inner.match(/<v>([\s\S]*?)<\/v>/) || [])[1];
|
|
192
|
+
if (v !== undefined) val = (t === 's') ? (shared[+v] ?? '') : v; // shared str vs number/formula
|
|
193
|
+
}
|
|
194
|
+
line[col] = val;
|
|
195
|
+
if (col > maxCol) maxCol = col;
|
|
196
|
+
}
|
|
197
|
+
if (rn > maxRow) maxRow = rn;
|
|
198
|
+
}
|
|
199
|
+
return {
|
|
200
|
+
rowCount: maxRow,
|
|
201
|
+
colCount: maxCol,
|
|
202
|
+
/** 1-based, Excel-style: at(12, 3) === cell C12. */
|
|
203
|
+
at(row, col) { return (cells[row] && cells[row][col]) ?? ''; },
|
|
204
|
+
/** the row's cells as a 1-based-ish object (col → value). */
|
|
205
|
+
row(row) { return cells[row] || {}; },
|
|
206
|
+
};
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* Resolve the workbook's real sheet names, in workbook order, to their part paths.
|
|
211
|
+
* workbook.xml gives name + r:id; workbook.xml.rels maps r:id → worksheets/sheetN.xml.
|
|
212
|
+
* @returns {Array<{name:string, path:string}>}
|
|
213
|
+
*/
|
|
214
|
+
function resolveSheets(files) {
|
|
215
|
+
const wbXml = (files.get('xl/workbook.xml') || Buffer.from('')).toString('utf8');
|
|
216
|
+
const relsXml = (files.get('xl/_rels/workbook.xml.rels') || Buffer.from('')).toString('utf8');
|
|
217
|
+
|
|
218
|
+
const rels = {}; // rId → target part path
|
|
219
|
+
for (const m of relsXml.matchAll(/<Relationship\b[^>]*>/g)) {
|
|
220
|
+
const id = (m[0].match(/Id="([^"]+)"/) || [])[1];
|
|
221
|
+
let target = (m[0].match(/Target="([^"]+)"/) || [])[1];
|
|
222
|
+
if (!id || !target) continue;
|
|
223
|
+
target = target.replace(/^\/?xl\//, '').replace(/^\.\//, '');
|
|
224
|
+
rels[id] = 'xl/' + target;
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
const out = [];
|
|
228
|
+
for (const m of wbXml.matchAll(/<sheet\b[^>]*\/?>/g)) {
|
|
229
|
+
const name = (m[0].match(/name="([^"]*)"/) || [])[1];
|
|
230
|
+
const rid = (m[0].match(/r:id="([^"]+)"/) || [])[1];
|
|
231
|
+
const path = rels[rid];
|
|
232
|
+
if (name && path && files.has(path)) out.push({ name: unescapeXml(name), path });
|
|
233
|
+
}
|
|
234
|
+
if (out.length) return out;
|
|
235
|
+
|
|
236
|
+
// Fallback: no workbook.xml — order sheetN.xml numerically, name them by index.
|
|
237
|
+
return [...files.keys()]
|
|
238
|
+
.filter(k => /^xl\/worksheets\/sheet\d+\.xml$/.test(k))
|
|
239
|
+
.sort((a, b) => (+a.match(/(\d+)/)[1]) - (+b.match(/(\d+)/)[1]))
|
|
240
|
+
.map((path, i) => ({ name: `Sheet${i + 1}`, path }));
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/**
|
|
244
|
+
* Read an .xlsx directly (path or Buffer). Unzip is built in.
|
|
245
|
+
* @returns {{sheetNames:string[], sheet:(nameOrIndex?:string|number)=>Sheet, sharedStrings:string[]}}
|
|
246
|
+
* `sheet()` accepts a real sheet name ("GL 24500 Accrued Bonus") or a 0-based index.
|
|
247
|
+
*/
|
|
248
|
+
function readXlsx(pathOrBuffer) {
|
|
249
|
+
const buf = Buffer.isBuffer(pathOrBuffer) ? pathOrBuffer : fs.readFileSync(pathOrBuffer);
|
|
250
|
+
const files = unzip(buf);
|
|
251
|
+
const ssXml = files.has('xl/sharedStrings.xml') ? files.get('xl/sharedStrings.xml').toString('utf8') : '';
|
|
252
|
+
const shared = parseSharedStrings(ssXml);
|
|
253
|
+
const sheets = resolveSheets(files);
|
|
254
|
+
const cache = {};
|
|
255
|
+
return {
|
|
256
|
+
sheetNames: sheets.map(s => s.name),
|
|
257
|
+
sharedStrings: shared,
|
|
258
|
+
sheet(nameOrIndex = 0) {
|
|
259
|
+
const i = (typeof nameOrIndex === 'number')
|
|
260
|
+
? nameOrIndex
|
|
261
|
+
: sheets.findIndex(s => s.name === nameOrIndex);
|
|
262
|
+
if (i < 0 || !sheets[i]) throw new Error('filekit.readXlsx: no sheet ' + JSON.stringify(nameOrIndex));
|
|
263
|
+
if (!cache[i]) cache[i] = makeSheet(files.get(sheets[i].path).toString('utf8'), shared);
|
|
264
|
+
return cache[i];
|
|
265
|
+
},
|
|
266
|
+
};
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
/* ------------------------------------------------------------------ *
|
|
270
|
+
* XLSX writing — generate a reference workbook to hand a client.
|
|
271
|
+
* Stored (uncompressed) zip + the minimal OOXML parts. No dependencies.
|
|
272
|
+
* ------------------------------------------------------------------ */
|
|
273
|
+
|
|
274
|
+
const CRC_TABLE = (() => {
|
|
275
|
+
const t = new Int32Array(256);
|
|
276
|
+
for (let n = 0; n < 256; n++) {
|
|
277
|
+
let c = n;
|
|
278
|
+
for (let k = 0; k < 8; k++) c = (c & 1) ? (0xedb88320 ^ (c >>> 1)) : (c >>> 1);
|
|
279
|
+
t[n] = c;
|
|
280
|
+
}
|
|
281
|
+
return t;
|
|
282
|
+
})();
|
|
283
|
+
|
|
284
|
+
function crc32(buf) {
|
|
285
|
+
if (typeof zlib.crc32 === 'function') return zlib.crc32(buf) >>> 0;
|
|
286
|
+
let c = -1;
|
|
287
|
+
for (let i = 0; i < buf.length; i++) c = CRC_TABLE[(c ^ buf[i]) & 0xff] ^ (c >>> 8);
|
|
288
|
+
return (c ^ -1) >>> 0;
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
/** 1-based column number → Excel letters (1 → A, 27 → AA). */
|
|
292
|
+
function numToCol(n) {
|
|
293
|
+
let s = '';
|
|
294
|
+
while (n > 0) { const m = (n - 1) % 26; s = String.fromCharCode(65 + m) + s; n = Math.floor((n - 1) / 26); }
|
|
295
|
+
return s;
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
function escXml(s) {
|
|
299
|
+
return String(s)
|
|
300
|
+
.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>')
|
|
301
|
+
.replace(/"/g, '"').replace(/\r/g, '')
|
|
302
|
+
// strip control chars XML 1.0 forbids
|
|
303
|
+
.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F]/g, '');
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
/**
|
|
307
|
+
* Build a ZIP from Map<name, Buffer>. Deterministic: fixed timestamps, and each
|
|
308
|
+
* entry is deflated when that shrinks it, else stored — so output is byte-stable
|
|
309
|
+
* for a given input and stays comparable in size to the file it replaces.
|
|
310
|
+
*/
|
|
311
|
+
function zipStore(files) {
|
|
312
|
+
const DOS_DATE = 33, DOS_TIME = 0; // 1980-01-01, so output is byte-stable
|
|
313
|
+
const locals = [], centrals = [];
|
|
314
|
+
let offset = 0;
|
|
315
|
+
|
|
316
|
+
for (const [name, data] of files) {
|
|
317
|
+
const nameBuf = Buffer.from(name, 'utf8');
|
|
318
|
+
const crc = crc32(data); // always over the UNCOMPRESSED bytes
|
|
319
|
+
|
|
320
|
+
const deflated = zlib.deflateRawSync(data, { level: 9 });
|
|
321
|
+
const useDeflate = deflated.length < data.length;
|
|
322
|
+
const method = useDeflate ? 8 : 0;
|
|
323
|
+
const payload = useDeflate ? deflated : data;
|
|
324
|
+
|
|
325
|
+
const local = Buffer.alloc(30);
|
|
326
|
+
local.writeUInt32LE(0x04034b50, 0);
|
|
327
|
+
local.writeUInt16LE(20, 4); // version needed
|
|
328
|
+
local.writeUInt16LE(0, 6); // flags
|
|
329
|
+
local.writeUInt16LE(method, 8);
|
|
330
|
+
local.writeUInt16LE(DOS_TIME, 10);
|
|
331
|
+
local.writeUInt16LE(DOS_DATE, 12);
|
|
332
|
+
local.writeUInt32LE(crc, 14);
|
|
333
|
+
local.writeUInt32LE(payload.length, 18); // compressed size
|
|
334
|
+
local.writeUInt32LE(data.length, 22); // uncompressed size
|
|
335
|
+
local.writeUInt16LE(nameBuf.length, 26);
|
|
336
|
+
local.writeUInt16LE(0, 28);
|
|
337
|
+
locals.push(local, nameBuf, payload);
|
|
338
|
+
|
|
339
|
+
const central = Buffer.alloc(46);
|
|
340
|
+
central.writeUInt32LE(0x02014b50, 0);
|
|
341
|
+
central.writeUInt16LE(20, 4); // version made by
|
|
342
|
+
central.writeUInt16LE(20, 6); // version needed
|
|
343
|
+
central.writeUInt16LE(0, 8);
|
|
344
|
+
central.writeUInt16LE(method, 10);
|
|
345
|
+
central.writeUInt16LE(DOS_TIME, 12);
|
|
346
|
+
central.writeUInt16LE(DOS_DATE, 14);
|
|
347
|
+
central.writeUInt32LE(crc, 16);
|
|
348
|
+
central.writeUInt32LE(payload.length, 20);
|
|
349
|
+
central.writeUInt32LE(data.length, 24);
|
|
350
|
+
central.writeUInt16LE(nameBuf.length, 28);
|
|
351
|
+
central.writeUInt16LE(0, 30); // extra
|
|
352
|
+
central.writeUInt16LE(0, 32); // comment
|
|
353
|
+
central.writeUInt16LE(0, 34); // disk
|
|
354
|
+
central.writeUInt16LE(0, 36); // internal attrs
|
|
355
|
+
central.writeUInt32LE(0, 38); // external attrs
|
|
356
|
+
central.writeUInt32LE(offset, 42);
|
|
357
|
+
centrals.push(central, nameBuf);
|
|
358
|
+
|
|
359
|
+
offset += local.length + nameBuf.length + payload.length;
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
const localPart = Buffer.concat(locals);
|
|
363
|
+
const centralPart = Buffer.concat(centrals);
|
|
364
|
+
const eocd = Buffer.alloc(22);
|
|
365
|
+
eocd.writeUInt32LE(0x06054b50, 0);
|
|
366
|
+
eocd.writeUInt16LE(0, 4);
|
|
367
|
+
eocd.writeUInt16LE(0, 6);
|
|
368
|
+
eocd.writeUInt16LE(files.size, 8);
|
|
369
|
+
eocd.writeUInt16LE(files.size, 10);
|
|
370
|
+
eocd.writeUInt32LE(centralPart.length, 12);
|
|
371
|
+
eocd.writeUInt32LE(localPart.length, 16);
|
|
372
|
+
eocd.writeUInt16LE(0, 20);
|
|
373
|
+
|
|
374
|
+
return Buffer.concat([localPart, centralPart, eocd]);
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
const SHEET_NAME_BAD = /[\[\]:*?/\\]/;
|
|
378
|
+
|
|
379
|
+
function sheetXml(rows) {
|
|
380
|
+
const out = [
|
|
381
|
+
'<?xml version="1.0" encoding="UTF-8" standalone="yes"?>',
|
|
382
|
+
'<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><sheetData>',
|
|
383
|
+
];
|
|
384
|
+
rows.forEach((row, r) => {
|
|
385
|
+
const cells = [];
|
|
386
|
+
row.forEach((v, c) => {
|
|
387
|
+
if (v === null || v === undefined || v === '') return; // empty cell = null on ingest
|
|
388
|
+
const ref = numToCol(c + 1) + (r + 1);
|
|
389
|
+
if (typeof v === 'number' && Number.isFinite(v)) {
|
|
390
|
+
cells.push(`<c r="${ref}"><v>${v}</v></c>`);
|
|
391
|
+
} else {
|
|
392
|
+
cells.push(`<c r="${ref}" t="inlineStr"><is><t xml:space="preserve">${escXml(v)}</t></is></c>`);
|
|
393
|
+
}
|
|
394
|
+
});
|
|
395
|
+
if (cells.length) out.push(`<row r="${r + 1}">${cells.join('')}</row>`);
|
|
396
|
+
});
|
|
397
|
+
out.push('</sheetData></worksheet>');
|
|
398
|
+
return Buffer.from(out.join(''), 'utf8');
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
/**
|
|
402
|
+
* Write a multi-tab .xlsx. One tab per dataset — the tab name becomes the bronze table suffix.
|
|
403
|
+
* @param {string|null} filePath write here, or null to only return the buffer
|
|
404
|
+
* @param {Array<{name:string, rows:Array<Array<string|number|null>>}>} sheets row 0 = header
|
|
405
|
+
* @returns {Buffer} the .xlsx bytes
|
|
406
|
+
*/
|
|
407
|
+
function writeXlsx(filePath, sheets) {
|
|
408
|
+
if (!sheets || !sheets.length) throw new Error('filekit.writeXlsx: need at least one sheet');
|
|
409
|
+
for (const s of sheets) {
|
|
410
|
+
if (!s.name || s.name.length > 31 || SHEET_NAME_BAD.test(s.name)) {
|
|
411
|
+
throw new Error('filekit.writeXlsx: invalid sheet name ' + JSON.stringify(s.name));
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
const names = sheets.map(s => s.name);
|
|
415
|
+
if (new Set(names).size !== names.length) throw new Error('filekit.writeXlsx: duplicate sheet names');
|
|
416
|
+
|
|
417
|
+
const files = new Map();
|
|
418
|
+
const stylesRid = 'rId' + (sheets.length + 1);
|
|
419
|
+
|
|
420
|
+
files.set('[Content_Types].xml', Buffer.from(
|
|
421
|
+
'<?xml version="1.0" encoding="UTF-8" standalone="yes"?>' +
|
|
422
|
+
'<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">' +
|
|
423
|
+
'<Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>' +
|
|
424
|
+
'<Default Extension="xml" ContentType="application/xml"/>' +
|
|
425
|
+
'<Override PartName="/xl/workbook.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"/>' +
|
|
426
|
+
sheets.map((_, i) => `<Override PartName="/xl/worksheets/sheet${i + 1}.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"/>`).join('') +
|
|
427
|
+
'<Override PartName="/xl/styles.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.styles+xml"/>' +
|
|
428
|
+
'</Types>', 'utf8'));
|
|
429
|
+
|
|
430
|
+
files.set('_rels/.rels', Buffer.from(
|
|
431
|
+
'<?xml version="1.0" encoding="UTF-8" standalone="yes"?>' +
|
|
432
|
+
'<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">' +
|
|
433
|
+
'<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="xl/workbook.xml"/>' +
|
|
434
|
+
'</Relationships>', 'utf8'));
|
|
435
|
+
|
|
436
|
+
files.set('xl/workbook.xml', Buffer.from(
|
|
437
|
+
'<?xml version="1.0" encoding="UTF-8" standalone="yes"?>' +
|
|
438
|
+
'<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" ' +
|
|
439
|
+
'xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"><sheets>' +
|
|
440
|
+
sheets.map((s, i) => `<sheet name="${escXml(s.name)}" sheetId="${i + 1}" r:id="rId${i + 1}"/>`).join('') +
|
|
441
|
+
'</sheets></workbook>', 'utf8'));
|
|
442
|
+
|
|
443
|
+
files.set('xl/_rels/workbook.xml.rels', Buffer.from(
|
|
444
|
+
'<?xml version="1.0" encoding="UTF-8" standalone="yes"?>' +
|
|
445
|
+
'<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">' +
|
|
446
|
+
sheets.map((_, i) => `<Relationship Id="rId${i + 1}" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="worksheets/sheet${i + 1}.xml"/>`).join('') +
|
|
447
|
+
`<Relationship Id="${stylesRid}" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/styles" Target="styles.xml"/>` +
|
|
448
|
+
'</Relationships>', 'utf8'));
|
|
449
|
+
|
|
450
|
+
files.set('xl/styles.xml', Buffer.from(
|
|
451
|
+
'<?xml version="1.0" encoding="UTF-8" standalone="yes"?>' +
|
|
452
|
+
'<styleSheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">' +
|
|
453
|
+
'<fonts count="1"><font><sz val="11"/><name val="Calibri"/></font></fonts>' +
|
|
454
|
+
'<fills count="2"><fill><patternFill patternType="none"/></fill><fill><patternFill patternType="gray125"/></fill></fills>' +
|
|
455
|
+
'<borders count="1"><border/></borders>' +
|
|
456
|
+
'<cellStyleXfs count="1"><xf numFmtId="0" fontId="0" fillId="0" borderId="0"/></cellStyleXfs>' +
|
|
457
|
+
'<cellXfs count="1"><xf numFmtId="0" fontId="0" fillId="0" borderId="0" xfId="0"/></cellXfs>' +
|
|
458
|
+
'</styleSheet>', 'utf8'));
|
|
459
|
+
|
|
460
|
+
sheets.forEach((s, i) => files.set(`xl/worksheets/sheet${i + 1}.xml`, sheetXml(s.rows)));
|
|
461
|
+
|
|
462
|
+
const buf = zipStore(files);
|
|
463
|
+
if (filePath) fs.writeFileSync(filePath, buf);
|
|
464
|
+
return buf;
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
/* ------------------------------------------------------------------ *
|
|
468
|
+
* Small grid helpers
|
|
469
|
+
* ------------------------------------------------------------------ */
|
|
470
|
+
|
|
471
|
+
/**
|
|
472
|
+
* Find the first Excel row (1-based) whose cells satisfy a predicate.
|
|
473
|
+
* @param {Sheet} sheet
|
|
474
|
+
* @param {(rowObj:object, rowNum:number)=>boolean} predicate
|
|
475
|
+
*/
|
|
476
|
+
function findRow(sheet, predicate) {
|
|
477
|
+
for (let r = 1; r <= sheet.rowCount; r++) {
|
|
478
|
+
if (predicate(sheet.row(r), r)) return r;
|
|
479
|
+
}
|
|
480
|
+
return -1;
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
/** In a header row of Excel serial dates, find the column whose date is `ym` ("YYYY-MM"). */
|
|
484
|
+
function findColumnBySerialMonth(sheet, headerRowNum, ym) {
|
|
485
|
+
const line = sheet.row(headerRowNum);
|
|
486
|
+
for (const col of Object.keys(line)) {
|
|
487
|
+
const n = num(line[col]);
|
|
488
|
+
if (n > 40000 && n < 60000 && excelSerialToYM(n) === ym) return +col;
|
|
489
|
+
}
|
|
490
|
+
return -1;
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
module.exports = {
|
|
494
|
+
num, excelSerialToDate, excelSerialToYM,
|
|
495
|
+
parseCsv, tokenizeCsv, findMonthColumns,
|
|
496
|
+
unzip, readXlsx, resolveSheets, findRow, findColumnBySerialMonth,
|
|
497
|
+
zipStore, writeXlsx, numToCol, crc32,
|
|
498
|
+
};
|
package/package.json
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@idea1/filekit",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"description": "Messy-file reader/writer toolkit (CSV + XLSX) for the close kit and similar agent-driven data pipelines. Pure Node, no dependencies.",
|
|
5
|
+
"main": "filekit.js",
|
|
6
|
+
"files": [
|
|
7
|
+
"filekit.js",
|
|
8
|
+
"README.md"
|
|
9
|
+
],
|
|
10
|
+
"engines": {
|
|
11
|
+
"node": ">=20"
|
|
12
|
+
},
|
|
13
|
+
"scripts": {
|
|
14
|
+
"test": "node filekit.test.js",
|
|
15
|
+
"build": "npm test",
|
|
16
|
+
"prepublishOnly": "npm run build",
|
|
17
|
+
"deploy": "npm publish --access public"
|
|
18
|
+
},
|
|
19
|
+
"repository": {
|
|
20
|
+
"type": "git",
|
|
21
|
+
"url": "git+https://github.com/datafabriq-co/datafabriq.git",
|
|
22
|
+
"directory": "packages/filekit"
|
|
23
|
+
},
|
|
24
|
+
"license": "MIT"
|
|
25
|
+
}
|