@chriswiegman/hugo-tools 0.2.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/README.md +28 -2
- package/package.json +2 -2
- package/src/book.mjs +616 -13
- package/src/book.test.mjs +621 -4
package/src/book.mjs
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
/**
|
|
3
|
-
* Goodreads → Hugo Books importer
|
|
3
|
+
* Goodreads → Hugo Books importer
|
|
4
4
|
* - Writes: content/books/<author-last-first>/<title-slug>.md
|
|
5
5
|
* - Front matter:
|
|
6
|
-
* title, author, rating (0-5), finished ([YYYY-MM-DD...]), links { amazon, openlibrary, goodreads }
|
|
6
|
+
* title, author, rating (0-5), finished ([YYYY-MM-DD...]), links { amazon, openlibrary, goodreads },
|
|
7
|
+
* reference { isbn, asin }, cover
|
|
7
8
|
* - Review (if any) becomes body content
|
|
8
9
|
* - finished uses Date Read; if blank, Date Added (only for Exclusive Shelf == "read")
|
|
9
10
|
*
|
|
@@ -12,11 +13,44 @@
|
|
|
12
13
|
* - Existing files are matched by Goodreads Book ID first, then ISBN, then title slug.
|
|
13
14
|
* - When a match is found at a different path (title changed in Goodreads), the file is
|
|
14
15
|
* renamed to the new slug and the title in front matter is updated automatically.
|
|
15
|
-
* - For an existing book file, finished dates are merged
|
|
16
|
+
* - For an existing book file, finished dates are merged and the rating is updated when the
|
|
17
|
+
* Goodreads rating differs from the file's rating — but only when Goodreads has a rating
|
|
18
|
+
* (> 0). A Goodreads rating of 0 (unrated) never overwrites a rating already in the file,
|
|
19
|
+
* since you may have rated the book only in the markdown. Other fields are left untouched
|
|
16
20
|
* unless a rename occurs (in which case the title is also updated).
|
|
21
|
+
* - reference.isbn/asin are backfilled onto existing files that are missing them, but a
|
|
22
|
+
* value already present in the file (e.g. entered by hand) is never overwritten.
|
|
23
|
+
*
|
|
24
|
+
* Re-reads:
|
|
25
|
+
* - Goodreads' CSV export only ever contains the *most recent* "Date Read" per book, even
|
|
26
|
+
* when "Read Count" shows it was finished more than once. So: if you re-read a book,
|
|
27
|
+
* the next export will carry the new completion date, and — since it differs from what's
|
|
28
|
+
* already on file — it's automatically merged into `finished` as an additional date
|
|
29
|
+
* (nothing is overwritten; dates only ever get added, never replaced or removed).
|
|
30
|
+
* - What the export can't give you is *earlier* read dates for a book already read more
|
|
31
|
+
* than once before this tool started tracking it. When "Read Count" is higher than the
|
|
32
|
+
* number of dates on file, the script prints a `NOTE:` line and leaves the file
|
|
33
|
+
* untouched — add the missing date(s) to `finished` by hand. The note stops appearing
|
|
34
|
+
* once the counts line up.
|
|
35
|
+
*
|
|
36
|
+
* ASIN lookup:
|
|
37
|
+
* - Goodreads exports don't include an ASIN, so it's looked up per ISBN from Open Library,
|
|
38
|
+
* then Google Books, and left blank if neither has one. Configure an Amazon Product
|
|
39
|
+
* Advertising API key in .hugo-tools.json (amazonPaApi: { accessKey, secretKey, partnerTag })
|
|
40
|
+
* to use it as a final fallback.
|
|
41
|
+
*
|
|
42
|
+
* Cover art (opt-in):
|
|
43
|
+
* - Off by default — pass --covers to fetch and save cover art.
|
|
44
|
+
* - Looked up per ISBN from Open Library's cover API, then Google Books, and left blank if
|
|
45
|
+
* neither has one.
|
|
46
|
+
* - Saved to <coversDir>/<author-slug>/<title-slug>.<ext> (coversDir defaults to
|
|
47
|
+
* assets/images/books, configurable via .hugo-tools.json) and referenced from the
|
|
48
|
+
* `cover` front matter field as a Hugo URL path.
|
|
49
|
+
* - Only backfills a blank `cover` field on existing files — a cover already set
|
|
50
|
+
* (e.g. entered by hand) is never overwritten.
|
|
17
51
|
*
|
|
18
52
|
* Usage:
|
|
19
|
-
* node scripts/book.mjs path/to/goodreads.csv
|
|
53
|
+
* node scripts/book.mjs path/to/goodreads.csv [--covers]
|
|
20
54
|
*
|
|
21
55
|
* Notes:
|
|
22
56
|
* - This script uses global fetch (Node 18+).
|
|
@@ -25,6 +59,7 @@
|
|
|
25
59
|
import fs from 'node:fs/promises';
|
|
26
60
|
import { existsSync, readFileSync } from 'node:fs';
|
|
27
61
|
import path from 'node:path';
|
|
62
|
+
import { createHash, createHmac } from 'node:crypto';
|
|
28
63
|
|
|
29
64
|
// -------------------- Config --------------------
|
|
30
65
|
|
|
@@ -42,10 +77,17 @@ try {
|
|
|
42
77
|
|
|
43
78
|
const OUTPUT_ROOT = path.join(PROJECT_ROOT, _userConfig.booksDir ?? 'content/books');
|
|
44
79
|
|
|
80
|
+
// Optional Amazon Product Advertising API credentials for ASIN lookup fallback.
|
|
81
|
+
// Set via .hugo-tools.json: { "amazonPaApi": { "accessKey", "secretKey", "partnerTag" } }
|
|
82
|
+
const AMAZON_PA_CONFIG = _userConfig.amazonPaApi ?? null;
|
|
83
|
+
|
|
84
|
+
// Project-relative directory cover art is saved under. Set via .hugo-tools.json: { "coversDir" }
|
|
85
|
+
const COVERS_DIR = _userConfig.coversDir ?? 'assets/images/books';
|
|
86
|
+
|
|
45
87
|
// -------------------- Utilities --------------------
|
|
46
88
|
|
|
47
89
|
function usageAndExit() {
|
|
48
|
-
console.error('Usage: node scripts/book.mjs path/to/goodreads.csv');
|
|
90
|
+
console.error('Usage: node scripts/book.mjs path/to/goodreads.csv [--covers]');
|
|
49
91
|
process.exit(1);
|
|
50
92
|
}
|
|
51
93
|
|
|
@@ -200,6 +242,24 @@ function keyFor(title, author) {
|
|
|
200
242
|
return `${title.trim().toLowerCase()}|${author.trim().toLowerCase()}`;
|
|
201
243
|
}
|
|
202
244
|
|
|
245
|
+
/**
|
|
246
|
+
* Strip a UTF-8 BOM if present. Some export tools/editors add one, and it would
|
|
247
|
+
* otherwise corrupt the first header name (e.g. "\uFEFFBook Id") and fail column detection.
|
|
248
|
+
*/
|
|
249
|
+
function stripBOM(text) {
|
|
250
|
+
return String(text ?? '').replace(/^\uFEFF/, '');
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
/**
|
|
254
|
+
* Goodreads' CSV export only ever includes the most recent "Date Read" per book, even
|
|
255
|
+
* when "Read Count" shows it was finished more than once — there's no way to recover
|
|
256
|
+
* earlier read dates from the export alone. Flag books where fewer dates are on record
|
|
257
|
+
* than the Goodreads read count, as a nudge to add the missing ones to `finished` by hand.
|
|
258
|
+
*/
|
|
259
|
+
function needsReReadDates(readCount, finishedCount) {
|
|
260
|
+
return Number.isFinite(readCount) && readCount > 1 && finishedCount < readCount;
|
|
261
|
+
}
|
|
262
|
+
|
|
203
263
|
function goodreadsBookUrl(bookId) {
|
|
204
264
|
return bookId ? `https://www.goodreads.com/book/show/${bookId}` : '';
|
|
205
265
|
}
|
|
@@ -215,6 +275,292 @@ function amazonSearchLink({ isbn13, isbn10, title, author }) {
|
|
|
215
275
|
return `https://www.amazon.com/s?k=${encodeURIComponent(query)}`;
|
|
216
276
|
}
|
|
217
277
|
|
|
278
|
+
// -------------------- ASIN lookup --------------------
|
|
279
|
+
|
|
280
|
+
function sleep(ms) {
|
|
281
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
/**
|
|
285
|
+
* Open Library sometimes exposes an "amazon" identifier on an edition, which
|
|
286
|
+
* for Kindle editions is the ASIN.
|
|
287
|
+
*/
|
|
288
|
+
async function lookupAsinFromOpenLibrary(isbn, fetchImpl = fetch) {
|
|
289
|
+
if (!isbn) return '';
|
|
290
|
+
|
|
291
|
+
try {
|
|
292
|
+
const url = `https://openlibrary.org/api/books?bibkeys=ISBN:${encodeURIComponent(isbn)}&jscmd=data&format=json`;
|
|
293
|
+
const res = await fetchImpl(url);
|
|
294
|
+
|
|
295
|
+
if (!res.ok) return '';
|
|
296
|
+
|
|
297
|
+
const data = await res.json();
|
|
298
|
+
const amazon = data?.[`ISBN:${isbn}`]?.identifiers?.amazon;
|
|
299
|
+
|
|
300
|
+
return Array.isArray(amazon) && amazon[0] ? String(amazon[0]).trim().toUpperCase() : '';
|
|
301
|
+
} catch {
|
|
302
|
+
return '';
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
/**
|
|
307
|
+
* Google Books occasionally lists a non-ISBN "OTHER" industry identifier that
|
|
308
|
+
* matches the shape of an ASIN. Best-effort only — most volumes won't have one.
|
|
309
|
+
*/
|
|
310
|
+
async function lookupAsinFromGoogleBooks(isbn, fetchImpl = fetch) {
|
|
311
|
+
if (!isbn) return '';
|
|
312
|
+
|
|
313
|
+
try {
|
|
314
|
+
const url = `https://www.googleapis.com/books/v1/volumes?q=isbn:${encodeURIComponent(isbn)}`;
|
|
315
|
+
const res = await fetchImpl(url);
|
|
316
|
+
|
|
317
|
+
if (!res.ok) return '';
|
|
318
|
+
|
|
319
|
+
const data = await res.json();
|
|
320
|
+
const identifiers = data?.items?.[0]?.volumeInfo?.industryIdentifiers ?? [];
|
|
321
|
+
|
|
322
|
+
for (const id of identifiers) {
|
|
323
|
+
if (id?.type !== 'OTHER') continue;
|
|
324
|
+
|
|
325
|
+
const bare = String(id.identifier || '').trim().toUpperCase().replace(/^[A-Z]+:/, '');
|
|
326
|
+
|
|
327
|
+
if (/^[A-Z0-9]{10}$/.test(bare) && bare !== isbn.toUpperCase()) return bare;
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
return '';
|
|
331
|
+
} catch {
|
|
332
|
+
return '';
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
function hmac(key, data) {
|
|
337
|
+
return createHmac('sha256', key).update(data, 'utf8').digest();
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
function sha256Hex(data) {
|
|
341
|
+
return createHash('sha256').update(data, 'utf8').digest('hex');
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
function amazonPaApiSignature({ secretKey, region, service, dateStamp, stringToSign }) {
|
|
345
|
+
const kDate = hmac(`AWS4${secretKey}`, dateStamp);
|
|
346
|
+
const kRegion = hmac(kDate, region);
|
|
347
|
+
const kService = hmac(kRegion, service);
|
|
348
|
+
const kSigning = hmac(kService, 'aws4_request');
|
|
349
|
+
|
|
350
|
+
return hmac(kSigning, stringToSign).toString('hex');
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
/**
|
|
354
|
+
* Official Amazon Product Advertising API v5 GetItems call, signed with SigV4.
|
|
355
|
+
* Only attempted when accessKey/secretKey/partnerTag are configured.
|
|
356
|
+
*/
|
|
357
|
+
async function lookupAsinFromAmazonPa(isbn, config, fetchImpl = fetch) {
|
|
358
|
+
if (!isbn || !config?.accessKey || !config?.secretKey || !config?.partnerTag) return '';
|
|
359
|
+
|
|
360
|
+
const host = config.host || 'webservices.amazon.com';
|
|
361
|
+
const region = config.region || 'us-east-1';
|
|
362
|
+
const marketplace = config.marketplace || 'www.amazon.com';
|
|
363
|
+
const service = 'ProductAdvertisingAPI';
|
|
364
|
+
const target = 'com.amazon.paapi5.v1.ProductAdvertisingAPIv1.GetItems';
|
|
365
|
+
const uri = '/paapi5/getitems';
|
|
366
|
+
|
|
367
|
+
const payload = JSON.stringify({
|
|
368
|
+
ItemIds: [isbn],
|
|
369
|
+
ItemIdType: 'ISBN',
|
|
370
|
+
Resources: ['ItemInfo.Title'],
|
|
371
|
+
PartnerTag: config.partnerTag,
|
|
372
|
+
PartnerType: 'Associates',
|
|
373
|
+
Marketplace: marketplace,
|
|
374
|
+
});
|
|
375
|
+
|
|
376
|
+
const amzDate = new Date().toISOString().replace(/[:-]|\.\d{3}/g, '');
|
|
377
|
+
const dateStamp = amzDate.slice(0, 8);
|
|
378
|
+
const canonicalHeaders =
|
|
379
|
+
`content-encoding:amz-1.0\ncontent-type:application/json; charset=utf-8\nhost:${host}\n` +
|
|
380
|
+
`x-amz-date:${amzDate}\nx-amz-target:${target}\n`;
|
|
381
|
+
const signedHeaders = 'content-encoding;content-type;host;x-amz-date;x-amz-target';
|
|
382
|
+
const canonicalRequest = `POST\n${uri}\n\n${canonicalHeaders}\n${signedHeaders}\n${sha256Hex(payload)}`;
|
|
383
|
+
const credentialScope = `${dateStamp}/${region}/${service}/aws4_request`;
|
|
384
|
+
const stringToSign =
|
|
385
|
+
`AWS4-HMAC-SHA256\n${amzDate}\n${credentialScope}\n${sha256Hex(canonicalRequest)}`;
|
|
386
|
+
const signature = amazonPaApiSignature({ secretKey: config.secretKey, region, service, dateStamp, stringToSign });
|
|
387
|
+
const authorization =
|
|
388
|
+
`AWS4-HMAC-SHA256 Credential=${config.accessKey}/${credentialScope}, ` +
|
|
389
|
+
`SignedHeaders=${signedHeaders}, Signature=${signature}`;
|
|
390
|
+
|
|
391
|
+
try {
|
|
392
|
+
const res = await fetchImpl(`https://${host}${uri}`, {
|
|
393
|
+
method: 'POST',
|
|
394
|
+
headers: {
|
|
395
|
+
'content-encoding': 'amz-1.0',
|
|
396
|
+
'content-type': 'application/json; charset=utf-8',
|
|
397
|
+
host,
|
|
398
|
+
'x-amz-date': amzDate,
|
|
399
|
+
'x-amz-target': target,
|
|
400
|
+
authorization,
|
|
401
|
+
},
|
|
402
|
+
body: payload,
|
|
403
|
+
});
|
|
404
|
+
|
|
405
|
+
if (!res.ok) return '';
|
|
406
|
+
|
|
407
|
+
const data = await res.json();
|
|
408
|
+
const asin = data?.ItemsResult?.Items?.[0]?.ASIN;
|
|
409
|
+
|
|
410
|
+
return asin ? String(asin).trim().toUpperCase() : '';
|
|
411
|
+
} catch {
|
|
412
|
+
return '';
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
/**
|
|
417
|
+
* Look up an ASIN for an ISBN, trying free sources first (Open Library, then
|
|
418
|
+
* Google Books) and falling back to the Amazon Product Advertising API only
|
|
419
|
+
* when it's configured. Returns '' when no source has a match.
|
|
420
|
+
*/
|
|
421
|
+
async function lookupAsin(isbn, amazonPaConfig, cache, fetchImpl = fetch) {
|
|
422
|
+
if (!isbn) return '';
|
|
423
|
+
|
|
424
|
+
if (cache?.has(isbn)) return cache.get(isbn);
|
|
425
|
+
|
|
426
|
+
let asin = await lookupAsinFromOpenLibrary(isbn, fetchImpl);
|
|
427
|
+
|
|
428
|
+
if (!asin) {
|
|
429
|
+
await sleep(150);
|
|
430
|
+
asin = await lookupAsinFromGoogleBooks(isbn, fetchImpl);
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
if (!asin && amazonPaConfig) {
|
|
434
|
+
await sleep(150);
|
|
435
|
+
asin = await lookupAsinFromAmazonPa(isbn, amazonPaConfig, fetchImpl);
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
if (cache) cache.set(isbn, asin);
|
|
439
|
+
|
|
440
|
+
return asin;
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
// -------------------- Cover art (opt-in) --------------------
|
|
444
|
+
|
|
445
|
+
const IMAGE_EXT_BY_CONTENT_TYPE = {
|
|
446
|
+
'image/jpeg': '.jpg',
|
|
447
|
+
'image/jpg': '.jpg',
|
|
448
|
+
'image/png': '.png',
|
|
449
|
+
'image/webp': '.webp',
|
|
450
|
+
'image/gif': '.gif',
|
|
451
|
+
};
|
|
452
|
+
|
|
453
|
+
function extFromContentType(contentType) {
|
|
454
|
+
const bare = String(contentType || '').split(';')[0].trim().toLowerCase();
|
|
455
|
+
|
|
456
|
+
return IMAGE_EXT_BY_CONTENT_TYPE[bare] || '.jpg';
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
/**
|
|
460
|
+
* Convert a project-relative asset path (e.g. "assets/images/books/king-stephen/it.jpg")
|
|
461
|
+
* into the Hugo-served URL path, mirroring pick-image's convention of stripping the
|
|
462
|
+
* `assets/` prefix.
|
|
463
|
+
*/
|
|
464
|
+
function coverUrlFromAssetPath(assetPath) {
|
|
465
|
+
const rel = assetPath.split(path.sep).join('/').replace(/^assets\//, '');
|
|
466
|
+
|
|
467
|
+
return `/${rel}`;
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
/**
|
|
471
|
+
* Open Library's cover API serves the image directly at a predictable URL keyed by
|
|
472
|
+
* ISBN. `default=false` makes it 404 instead of returning a placeholder when there's
|
|
473
|
+
* no cover, so a non-OK response reliably means "no cover here."
|
|
474
|
+
*/
|
|
475
|
+
async function fetchOpenLibraryCover(isbn, fetchImpl = fetch) {
|
|
476
|
+
if (!isbn) return null;
|
|
477
|
+
|
|
478
|
+
try {
|
|
479
|
+
const url = `https://covers.openlibrary.org/b/isbn/${encodeURIComponent(isbn)}-L.jpg?default=false`;
|
|
480
|
+
const res = await fetchImpl(url);
|
|
481
|
+
|
|
482
|
+
if (!res.ok) return null;
|
|
483
|
+
|
|
484
|
+
const contentType = res.headers?.get?.('content-type') || 'image/jpeg';
|
|
485
|
+
|
|
486
|
+
if (!contentType.startsWith('image/')) return null;
|
|
487
|
+
|
|
488
|
+
const buffer = Buffer.from(await res.arrayBuffer());
|
|
489
|
+
|
|
490
|
+
if (buffer.length === 0) return null;
|
|
491
|
+
|
|
492
|
+
return { buffer, contentType };
|
|
493
|
+
} catch {
|
|
494
|
+
return null;
|
|
495
|
+
}
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
/**
|
|
499
|
+
* Fallback cover source: Google Books' volume thumbnail, used when Open Library has none.
|
|
500
|
+
*/
|
|
501
|
+
async function fetchGoogleBooksCover(isbn, fetchImpl = fetch) {
|
|
502
|
+
if (!isbn) return null;
|
|
503
|
+
|
|
504
|
+
try {
|
|
505
|
+
const url = `https://www.googleapis.com/books/v1/volumes?q=isbn:${encodeURIComponent(isbn)}`;
|
|
506
|
+
const res = await fetchImpl(url);
|
|
507
|
+
|
|
508
|
+
if (!res.ok) return null;
|
|
509
|
+
|
|
510
|
+
const data = await res.json();
|
|
511
|
+
const imageLinks = data?.items?.[0]?.volumeInfo?.imageLinks;
|
|
512
|
+
const thumbnail = imageLinks?.thumbnail || imageLinks?.smallThumbnail;
|
|
513
|
+
|
|
514
|
+
if (!thumbnail) return null;
|
|
515
|
+
|
|
516
|
+
const imgRes = await fetchImpl(thumbnail.replace(/^http:/, 'https:'));
|
|
517
|
+
|
|
518
|
+
if (!imgRes.ok) return null;
|
|
519
|
+
|
|
520
|
+
const contentType = imgRes.headers?.get?.('content-type') || 'image/jpeg';
|
|
521
|
+
|
|
522
|
+
if (!contentType.startsWith('image/')) return null;
|
|
523
|
+
|
|
524
|
+
const buffer = Buffer.from(await imgRes.arrayBuffer());
|
|
525
|
+
|
|
526
|
+
if (buffer.length === 0) return null;
|
|
527
|
+
|
|
528
|
+
return { buffer, contentType };
|
|
529
|
+
} catch {
|
|
530
|
+
return null;
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
/**
|
|
535
|
+
* Look up cover art for an ISBN, trying Open Library first and Google Books second.
|
|
536
|
+
* Returns null when neither source has one.
|
|
537
|
+
*/
|
|
538
|
+
async function fetchBookCover(isbn, fetchImpl = fetch) {
|
|
539
|
+
if (!isbn) return null;
|
|
540
|
+
|
|
541
|
+
const openLibrary = await fetchOpenLibraryCover(isbn, fetchImpl);
|
|
542
|
+
|
|
543
|
+
if (openLibrary) return openLibrary;
|
|
544
|
+
|
|
545
|
+
await sleep(150);
|
|
546
|
+
return fetchGoogleBooksCover(isbn, fetchImpl);
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
/**
|
|
550
|
+
* Write cover image bytes to `<coversDir>/<authorDir>/<slug><ext>` (relative to
|
|
551
|
+
* `projectRoot`) and return the Hugo URL path to store in front matter.
|
|
552
|
+
*/
|
|
553
|
+
async function saveCover(coversDir, authorDir, slug, cover, projectRoot = PROJECT_ROOT) {
|
|
554
|
+
const ext = extFromContentType(cover.contentType);
|
|
555
|
+
const relPath = path.join(coversDir, authorDir, `${slug}${ext}`);
|
|
556
|
+
const absPath = path.join(projectRoot, relPath);
|
|
557
|
+
|
|
558
|
+
await fs.mkdir(path.dirname(absPath), { recursive: true });
|
|
559
|
+
await fs.writeFile(absPath, cover.buffer);
|
|
560
|
+
|
|
561
|
+
return coverUrlFromAssetPath(relPath);
|
|
562
|
+
}
|
|
563
|
+
|
|
218
564
|
/**
|
|
219
565
|
* Use "Author l-f" (e.g. "Baldacci, David") → folder "baldacci-david"
|
|
220
566
|
*/
|
|
@@ -244,11 +590,13 @@ function authorDirFromAuthorLF(authorLF) {
|
|
|
244
590
|
* - "YYYY-MM-DD"
|
|
245
591
|
* - "YYYY-MM-DD"
|
|
246
592
|
* - finished: "YYYY-MM-DD" (legacy scalar; included as one date)
|
|
593
|
+
*
|
|
594
|
+
* Also extracts the existing `rating:` value (or null if absent/unparseable).
|
|
247
595
|
*/
|
|
248
596
|
function parseExistingMarkdown(md) {
|
|
249
597
|
const m = md.match(/^---\s*\n([\s\S]*?)\n---\s*\n?([\s\S]*)$/);
|
|
250
598
|
|
|
251
|
-
if (!m) return { finished: [] };
|
|
599
|
+
if (!m) return { finished: [], rating: null };
|
|
252
600
|
|
|
253
601
|
const fm = m[1];
|
|
254
602
|
const lines = fm.split('\n');
|
|
@@ -280,7 +628,18 @@ function parseExistingMarkdown(md) {
|
|
|
280
628
|
}
|
|
281
629
|
}
|
|
282
630
|
|
|
283
|
-
|
|
631
|
+
let rating = null;
|
|
632
|
+
|
|
633
|
+
for (const line of lines) {
|
|
634
|
+
const mm = line.match(/^rating:\s*(\d+)\s*$/);
|
|
635
|
+
|
|
636
|
+
if (mm) {
|
|
637
|
+
rating = Number(mm[1]);
|
|
638
|
+
break;
|
|
639
|
+
}
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
return { finished: uniqSortedDates(finished), rating };
|
|
284
643
|
}
|
|
285
644
|
|
|
286
645
|
async function fileExists(p) {
|
|
@@ -334,6 +693,116 @@ function updateTitleInFrontMatter(md, newTitle) {
|
|
|
334
693
|
return md.replace(/^title:.*$/m, `title: "${escapeYAMLString(newTitle)}"`);
|
|
335
694
|
}
|
|
336
695
|
|
|
696
|
+
/**
|
|
697
|
+
* Update the rating field in front matter, leaving everything else untouched.
|
|
698
|
+
*/
|
|
699
|
+
function updateRatingInFrontMatter(md, newRating) {
|
|
700
|
+
return md.replace(/^rating:.*$/m, `rating: ${newRating}`);
|
|
701
|
+
}
|
|
702
|
+
|
|
703
|
+
/**
|
|
704
|
+
* Extract the existing `reference: { isbn, asin }` values from a file's front matter.
|
|
705
|
+
* Returns { isbn: '', asin: '' } when the block (or either field) is absent.
|
|
706
|
+
*/
|
|
707
|
+
function parseExistingReference(md) {
|
|
708
|
+
const m = md.match(/^---\s*\n([\s\S]*?)\n---\s*\n?([\s\S]*)$/);
|
|
709
|
+
|
|
710
|
+
if (!m) return { isbn: '', asin: '' };
|
|
711
|
+
|
|
712
|
+
const lines = m[1].split('\n');
|
|
713
|
+
let isbn = '';
|
|
714
|
+
let asin = '';
|
|
715
|
+
|
|
716
|
+
for (let i = 0; i < lines.length; i++) {
|
|
717
|
+
if (lines[i].trim() !== 'reference:') continue;
|
|
718
|
+
|
|
719
|
+
for (let j = i + 1; j < lines.length; j++) {
|
|
720
|
+
const line = lines[j];
|
|
721
|
+
|
|
722
|
+
if (/^[A-Za-z0-9_-]+:/.test(line)) break; // new top-level key
|
|
723
|
+
|
|
724
|
+
const isbnMatch = line.match(/^\s*isbn:\s*"?([^"\n]*)"?\s*$/);
|
|
725
|
+
|
|
726
|
+
if (isbnMatch) isbn = isbnMatch[1].trim();
|
|
727
|
+
|
|
728
|
+
const asinMatch = line.match(/^\s*asin:\s*"?([^"\n]*)"?\s*$/);
|
|
729
|
+
|
|
730
|
+
if (asinMatch) asin = asinMatch[1].trim();
|
|
731
|
+
}
|
|
732
|
+
break;
|
|
733
|
+
}
|
|
734
|
+
|
|
735
|
+
return { isbn, asin };
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
/**
|
|
739
|
+
* Insert or replace the `reference:` block in front matter, leaving everything
|
|
740
|
+
* else untouched. Inserts a new block just before the closing `---` when none exists.
|
|
741
|
+
*/
|
|
742
|
+
function upsertReferenceInFrontMatter(md, { isbn, asin }) {
|
|
743
|
+
const block = [
|
|
744
|
+
'reference:',
|
|
745
|
+
` isbn: "${escapeYAMLString(isbn)}"`,
|
|
746
|
+
` asin: "${escapeYAMLString(asin)}"`,
|
|
747
|
+
];
|
|
748
|
+
const lines = md.split('\n');
|
|
749
|
+
const refIdx = lines.findIndex((l) => /^reference:\s*$/.test(l));
|
|
750
|
+
|
|
751
|
+
if (refIdx !== -1) {
|
|
752
|
+
let endIdx = refIdx + 1;
|
|
753
|
+
|
|
754
|
+
while (endIdx < lines.length && /^\s/.test(lines[endIdx])) {
|
|
755
|
+
endIdx++;
|
|
756
|
+
}
|
|
757
|
+
return [...lines.slice(0, refIdx), ...block, ...lines.slice(endIdx)].join('\n');
|
|
758
|
+
}
|
|
759
|
+
|
|
760
|
+
const closingIdx = lines.findIndex((l, i) => i > 0 && l.trim() === '---');
|
|
761
|
+
|
|
762
|
+
if (closingIdx === -1) {
|
|
763
|
+
console.warn(' WARNING: could not locate front matter to insert \'reference:\' block — file left unchanged');
|
|
764
|
+
return md;
|
|
765
|
+
}
|
|
766
|
+
|
|
767
|
+
return [...lines.slice(0, closingIdx), ...block, ...lines.slice(closingIdx)].join('\n');
|
|
768
|
+
}
|
|
769
|
+
|
|
770
|
+
/**
|
|
771
|
+
* Extract the existing scalar `cover:` value from a file's front matter.
|
|
772
|
+
* Returns '' when the field is absent.
|
|
773
|
+
*/
|
|
774
|
+
function parseExistingCover(md) {
|
|
775
|
+
const m = md.match(/^---\s*\n([\s\S]*?)\n---\s*\n?([\s\S]*)$/);
|
|
776
|
+
|
|
777
|
+
if (!m) return '';
|
|
778
|
+
|
|
779
|
+
const mm = m[1].match(/^cover:\s*"?([^"\n]*)"?\s*$/m);
|
|
780
|
+
|
|
781
|
+
return mm ? mm[1].trim() : '';
|
|
782
|
+
}
|
|
783
|
+
|
|
784
|
+
/**
|
|
785
|
+
* Insert or replace the scalar `cover:` field in front matter, leaving everything
|
|
786
|
+
* else untouched. Inserts a new line just before the closing `---` when absent.
|
|
787
|
+
*/
|
|
788
|
+
function upsertCoverInFrontMatter(md, coverPath) {
|
|
789
|
+
const line = `cover: "${escapeYAMLString(coverPath)}"`;
|
|
790
|
+
|
|
791
|
+
if (/^cover:.*$/m.test(md)) {
|
|
792
|
+
return md.replace(/^cover:.*$/m, line);
|
|
793
|
+
}
|
|
794
|
+
|
|
795
|
+
const lines = md.split('\n');
|
|
796
|
+
const closingIdx = lines.findIndex((l, i) => i > 0 && l.trim() === '---');
|
|
797
|
+
|
|
798
|
+
if (closingIdx === -1) {
|
|
799
|
+
console.warn(' WARNING: could not locate front matter to insert \'cover:\' field — file left unchanged');
|
|
800
|
+
return md;
|
|
801
|
+
}
|
|
802
|
+
|
|
803
|
+
return [...lines.slice(0, closingIdx), line, ...lines.slice(closingIdx)].join('\n');
|
|
804
|
+
}
|
|
805
|
+
|
|
337
806
|
// -------------------- Existing-file index --------------------
|
|
338
807
|
|
|
339
808
|
/**
|
|
@@ -462,7 +931,7 @@ async function removeIfEmpty(dirPath) {
|
|
|
462
931
|
|
|
463
932
|
// -------------------- Writing front matter --------------------
|
|
464
933
|
|
|
465
|
-
function toFrontMatter({ title, author, rating, finished, links }) {
|
|
934
|
+
function toFrontMatter({ title, author, rating, finished, links, reference, cover }) {
|
|
466
935
|
const lines = [];
|
|
467
936
|
|
|
468
937
|
lines.push('---');
|
|
@@ -476,6 +945,13 @@ function toFrontMatter({ title, author, rating, finished, links }) {
|
|
|
476
945
|
lines.push(` amazon: "${escapeYAMLString(links.amazon)}"`);
|
|
477
946
|
lines.push(` openlibrary: "${escapeYAMLString(links.openlibrary)}"`);
|
|
478
947
|
lines.push(` goodreads: "${escapeYAMLString(links.goodreads)}"`);
|
|
948
|
+
|
|
949
|
+
lines.push('reference:');
|
|
950
|
+
lines.push(` isbn: "${escapeYAMLString(reference?.isbn ?? '')}"`);
|
|
951
|
+
lines.push(` asin: "${escapeYAMLString(reference?.asin ?? '')}"`);
|
|
952
|
+
|
|
953
|
+
lines.push(`cover: "${escapeYAMLString(cover ?? '')}"`);
|
|
954
|
+
|
|
479
955
|
lines.push('---');
|
|
480
956
|
return lines.join('\n');
|
|
481
957
|
}
|
|
@@ -494,11 +970,13 @@ async function main() {
|
|
|
494
970
|
process.exit(1);
|
|
495
971
|
}
|
|
496
972
|
|
|
497
|
-
const
|
|
973
|
+
const argv = process.argv.slice(2);
|
|
974
|
+
const withCovers = argv.includes('--covers');
|
|
975
|
+
const csvPath = argv.find((a) => a && !a.startsWith('--'));
|
|
498
976
|
|
|
499
977
|
if (!csvPath) usageAndExit();
|
|
500
978
|
|
|
501
|
-
const csvText = await fs.readFile(csvPath, 'utf8');
|
|
979
|
+
const csvText = stripBOM(await fs.readFile(csvPath, 'utf8'));
|
|
502
980
|
const rows = parseCSV(csvText);
|
|
503
981
|
|
|
504
982
|
if (rows.length < 2) {
|
|
@@ -638,11 +1116,18 @@ async function main() {
|
|
|
638
1116
|
let mergedExisting = 0;
|
|
639
1117
|
let renamedFiles = 0;
|
|
640
1118
|
let skippedUnchanged = 0;
|
|
1119
|
+
let ratingsUpdated = 0;
|
|
1120
|
+
let referencesUpdated = 0;
|
|
1121
|
+
let coversFetched = 0;
|
|
1122
|
+
let reReadsFlagged = 0;
|
|
1123
|
+
|
|
1124
|
+
const asinCache = new Map();
|
|
641
1125
|
|
|
642
1126
|
for (const b of bookList) {
|
|
643
1127
|
const authorDir = authorDirFromAuthorLF(b.authorLF);
|
|
644
1128
|
const filename = `${slugify(b.title)}.md`;
|
|
645
1129
|
const outPath = path.join(OUTPUT_ROOT, authorDir, filename);
|
|
1130
|
+
const isbnBest = b.isbn13 || b.isbn10;
|
|
646
1131
|
|
|
647
1132
|
const existingPath = await findExistingFile(b, outPath, index);
|
|
648
1133
|
|
|
@@ -651,14 +1136,56 @@ async function main() {
|
|
|
651
1136
|
|
|
652
1137
|
// Read the existing file
|
|
653
1138
|
const existingContent = await fs.readFile(existingPath, 'utf8');
|
|
654
|
-
const { finished: existingFinished } = parseExistingMarkdown(existingContent);
|
|
1139
|
+
const { finished: existingFinished, rating: existingRating } = parseExistingMarkdown(existingContent);
|
|
655
1140
|
const finishedMerged = uniqSortedDates([...existingFinished, ...b.finished]);
|
|
656
1141
|
|
|
657
1142
|
const hasNewDates =
|
|
658
1143
|
finishedMerged.length !== existingFinished.length ||
|
|
659
1144
|
finishedMerged.some((d, i) => d !== existingFinished[i]);
|
|
660
1145
|
|
|
661
|
-
|
|
1146
|
+
// Only let Goodreads override the file's rating when Goodreads actually has one (> 0).
|
|
1147
|
+
// A rating of 0 usually means "not rated on Goodreads" — the file may carry a rating
|
|
1148
|
+
// entered by hand there instead, so leave it alone rather than clobbering it with 0.
|
|
1149
|
+
const hasRatingChange = b.rating > 0 && existingRating !== null && b.rating !== existingRating;
|
|
1150
|
+
|
|
1151
|
+
// Never clobber an isbn/asin already present in the file — only fill blanks.
|
|
1152
|
+
const { isbn: existingIsbn, asin: existingAsin } = parseExistingReference(existingContent);
|
|
1153
|
+
const finalIsbn = existingIsbn || isbnBest || '';
|
|
1154
|
+
let finalAsin = existingAsin;
|
|
1155
|
+
|
|
1156
|
+
if (!finalAsin && finalIsbn) {
|
|
1157
|
+
finalAsin = await lookupAsin(finalIsbn, AMAZON_PA_CONFIG, asinCache);
|
|
1158
|
+
}
|
|
1159
|
+
|
|
1160
|
+
const hasReferenceChange = finalIsbn !== existingIsbn || finalAsin !== existingAsin;
|
|
1161
|
+
|
|
1162
|
+
// Only fetch/backfill cover art when explicitly requested, and only into a blank field.
|
|
1163
|
+
const existingCover = parseExistingCover(existingContent);
|
|
1164
|
+
let finalCover = existingCover;
|
|
1165
|
+
|
|
1166
|
+
if (withCovers && !finalCover && finalIsbn) {
|
|
1167
|
+
const coverResult = await fetchBookCover(finalIsbn);
|
|
1168
|
+
|
|
1169
|
+
if (coverResult) {
|
|
1170
|
+
finalCover = await saveCover(COVERS_DIR, authorDir, slugify(b.title), coverResult);
|
|
1171
|
+
}
|
|
1172
|
+
}
|
|
1173
|
+
|
|
1174
|
+
const hasCoverChange = finalCover !== existingCover;
|
|
1175
|
+
|
|
1176
|
+
// Independent of whether anything else changed — surfaced every run until resolved.
|
|
1177
|
+
if (needsReReadDates(b.readCount, finishedMerged.length)) {
|
|
1178
|
+
reReadsFlagged++;
|
|
1179
|
+
|
|
1180
|
+
const rel = path.relative(OUTPUT_ROOT, existingPath);
|
|
1181
|
+
|
|
1182
|
+
console.log(
|
|
1183
|
+
` NOTE: ${rel} shows ${b.readCount}x read on Goodreads but only ${finishedMerged.length} ` +
|
|
1184
|
+
'date(s) on file — Goodreads only exports the latest read date, so add earlier ones to \'finished\' by hand.',
|
|
1185
|
+
);
|
|
1186
|
+
}
|
|
1187
|
+
|
|
1188
|
+
if (!titleChanged && !hasNewDates && !hasRatingChange && !hasReferenceChange && !hasCoverChange) {
|
|
662
1189
|
skippedUnchanged++;
|
|
663
1190
|
continue;
|
|
664
1191
|
}
|
|
@@ -670,6 +1197,33 @@ async function main() {
|
|
|
670
1197
|
patched = replaceFinishedBlock(patched, finishedMerged);
|
|
671
1198
|
}
|
|
672
1199
|
|
|
1200
|
+
if (hasRatingChange) {
|
|
1201
|
+
patched = updateRatingInFrontMatter(patched, b.rating);
|
|
1202
|
+
ratingsUpdated++;
|
|
1203
|
+
|
|
1204
|
+
const rel = path.relative(OUTPUT_ROOT, existingPath);
|
|
1205
|
+
|
|
1206
|
+
console.log(` RATING UPDATED: ${rel} (${existingRating} -> ${b.rating})`);
|
|
1207
|
+
}
|
|
1208
|
+
|
|
1209
|
+
if (hasReferenceChange) {
|
|
1210
|
+
patched = upsertReferenceInFrontMatter(patched, { isbn: finalIsbn, asin: finalAsin });
|
|
1211
|
+
referencesUpdated++;
|
|
1212
|
+
|
|
1213
|
+
const rel = path.relative(OUTPUT_ROOT, existingPath);
|
|
1214
|
+
|
|
1215
|
+
console.log(` REFERENCE UPDATED: ${rel} (isbn: "${finalIsbn}", asin: "${finalAsin}")`);
|
|
1216
|
+
}
|
|
1217
|
+
|
|
1218
|
+
if (hasCoverChange) {
|
|
1219
|
+
patched = upsertCoverInFrontMatter(patched, finalCover);
|
|
1220
|
+
coversFetched++;
|
|
1221
|
+
|
|
1222
|
+
const rel = path.relative(OUTPUT_ROOT, existingPath);
|
|
1223
|
+
|
|
1224
|
+
console.log(` COVER ADDED: ${rel} (${finalCover})`);
|
|
1225
|
+
}
|
|
1226
|
+
|
|
673
1227
|
if (titleChanged) {
|
|
674
1228
|
patched = updateTitleInFrontMatter(patched, b.title);
|
|
675
1229
|
}
|
|
@@ -698,7 +1252,15 @@ async function main() {
|
|
|
698
1252
|
createdNew++;
|
|
699
1253
|
|
|
700
1254
|
const finishedMerged = uniqSortedDates(b.finished);
|
|
701
|
-
|
|
1255
|
+
|
|
1256
|
+
if (needsReReadDates(b.readCount, finishedMerged.length)) {
|
|
1257
|
+
reReadsFlagged++;
|
|
1258
|
+
|
|
1259
|
+
console.log(
|
|
1260
|
+
` NOTE: "${b.title}" shows ${b.readCount}x read on Goodreads but only ${finishedMerged.length} ` +
|
|
1261
|
+
'date(s) on file — Goodreads only exports the latest read date, so add earlier ones to \'finished\' by hand.',
|
|
1262
|
+
);
|
|
1263
|
+
}
|
|
702
1264
|
|
|
703
1265
|
const links = {
|
|
704
1266
|
amazon: amazonSearchLink({ isbn13: b.isbn13, isbn10: b.isbn10, title: b.title, author: b.author }),
|
|
@@ -706,12 +1268,32 @@ async function main() {
|
|
|
706
1268
|
goodreads: goodreadsBookUrl(b.bookId),
|
|
707
1269
|
};
|
|
708
1270
|
|
|
1271
|
+
const reference = {
|
|
1272
|
+
isbn: isbnBest,
|
|
1273
|
+
asin: isbnBest ? await lookupAsin(isbnBest, AMAZON_PA_CONFIG, asinCache) : '',
|
|
1274
|
+
};
|
|
1275
|
+
|
|
1276
|
+
let cover = '';
|
|
1277
|
+
|
|
1278
|
+
if (withCovers && isbnBest) {
|
|
1279
|
+
const coverResult = await fetchBookCover(isbnBest);
|
|
1280
|
+
|
|
1281
|
+
if (coverResult) {
|
|
1282
|
+
cover = await saveCover(COVERS_DIR, authorDir, slugify(b.title), coverResult);
|
|
1283
|
+
coversFetched++;
|
|
1284
|
+
|
|
1285
|
+
console.log(` COVER SAVED: ${cover}`);
|
|
1286
|
+
}
|
|
1287
|
+
}
|
|
1288
|
+
|
|
709
1289
|
const fm = toFrontMatter({
|
|
710
1290
|
title: b.title,
|
|
711
1291
|
author: b.author,
|
|
712
1292
|
rating: b.rating,
|
|
713
1293
|
finished: finishedMerged,
|
|
714
1294
|
links,
|
|
1295
|
+
reference,
|
|
1296
|
+
cover,
|
|
715
1297
|
});
|
|
716
1298
|
|
|
717
1299
|
const bodyToWrite = b.review ? `\n\n${b.review}\n` : '\n';
|
|
@@ -746,7 +1328,11 @@ async function main() {
|
|
|
746
1328
|
` New files created: ${createdNew}\n` +
|
|
747
1329
|
` Existing files updated: ${mergedExisting}\n` +
|
|
748
1330
|
` Files renamed: ${renamedFiles}\n` +
|
|
1331
|
+
` Ratings updated: ${ratingsUpdated}\n` +
|
|
1332
|
+
` References updated: ${referencesUpdated}\n` +
|
|
1333
|
+
` Covers fetched: ${coversFetched}${withCovers ? '' : ' (pass --covers to fetch cover art)'}\n` +
|
|
749
1334
|
` Existing files skipped: ${skippedUnchanged}\n` +
|
|
1335
|
+
` Rereads needing dates: ${reReadsFlagged}${reReadsFlagged ? ' (see NOTE lines above)' : ''}\n` +
|
|
750
1336
|
` Content output: ${OUTPUT_ROOT}\n`,
|
|
751
1337
|
);
|
|
752
1338
|
}
|
|
@@ -771,6 +1357,8 @@ export {
|
|
|
771
1357
|
uniqSortedDates,
|
|
772
1358
|
authorDirFromAuthorLF,
|
|
773
1359
|
keyFor,
|
|
1360
|
+
needsReReadDates,
|
|
1361
|
+
stripBOM,
|
|
774
1362
|
goodreadsBookUrl,
|
|
775
1363
|
openLibraryIsbnUrl,
|
|
776
1364
|
amazonSearchLink,
|
|
@@ -778,9 +1366,24 @@ export {
|
|
|
778
1366
|
parseExistingMarkdown,
|
|
779
1367
|
replaceFinishedBlock,
|
|
780
1368
|
updateTitleInFrontMatter,
|
|
1369
|
+
updateRatingInFrontMatter,
|
|
1370
|
+
parseExistingReference,
|
|
1371
|
+
upsertReferenceInFrontMatter,
|
|
1372
|
+
parseExistingCover,
|
|
1373
|
+
upsertCoverInFrontMatter,
|
|
781
1374
|
extractGoodreadsId,
|
|
782
1375
|
extractFileIsbns,
|
|
783
1376
|
toFrontMatter,
|
|
784
1377
|
buildExistingIndex,
|
|
785
1378
|
findExistingFile,
|
|
1379
|
+
lookupAsinFromOpenLibrary,
|
|
1380
|
+
lookupAsinFromGoogleBooks,
|
|
1381
|
+
lookupAsinFromAmazonPa,
|
|
1382
|
+
lookupAsin,
|
|
1383
|
+
extFromContentType,
|
|
1384
|
+
coverUrlFromAssetPath,
|
|
1385
|
+
fetchOpenLibraryCover,
|
|
1386
|
+
fetchGoogleBooksCover,
|
|
1387
|
+
fetchBookCover,
|
|
1388
|
+
saveCover,
|
|
786
1389
|
};
|