@s-hirano-ist/s-scripts 1.25.2 → 1.26.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/README.md +73 -0
  2. package/configs/oxfmt.json +9 -0
  3. package/dist/cleanup-minio-images.js +11 -11
  4. package/dist/cleanup-minio-images.js.map +1 -1
  5. package/dist/enrich-books.js +6 -8
  6. package/dist/enrich-books.js.map +1 -1
  7. package/dist/fetch-articles.js +1 -2
  8. package/dist/fetch-articles.js.map +1 -1
  9. package/dist/fetch-books.js +3 -5
  10. package/dist/fetch-books.js.map +1 -1
  11. package/dist/fetch-images.js +3 -5
  12. package/dist/fetch-images.js.map +1 -1
  13. package/dist/fetch-notes.js +1 -2
  14. package/dist/fetch-notes.js.map +1 -1
  15. package/dist/fix-book-frontmatter.js +4 -4
  16. package/dist/fix-book-frontmatter.js.map +1 -1
  17. package/dist/fix-image-formats.js +2 -2
  18. package/dist/fix-image-formats.js.map +1 -1
  19. package/dist/infrastructures/articles-command-repository.d.ts +11 -11
  20. package/dist/infrastructures/articles-command-repository.d.ts.map +1 -1
  21. package/dist/infrastructures/books-command-repository.d.ts +11 -11
  22. package/dist/infrastructures/books-command-repository.d.ts.map +1 -1
  23. package/dist/infrastructures/cache-invalidation-client.d.ts +1 -1
  24. package/dist/infrastructures/cache-invalidation-client.d.ts.map +1 -1
  25. package/dist/infrastructures/images-command-repository.d.ts +11 -11
  26. package/dist/infrastructures/images-command-repository.d.ts.map +1 -1
  27. package/dist/infrastructures/notes-command-repository.d.ts +11 -11
  28. package/dist/infrastructures/notes-command-repository.d.ts.map +1 -1
  29. package/dist/ingest-articles.js +3 -4
  30. package/dist/ingest-articles.js.map +1 -1
  31. package/dist/ingest-books.js +7 -8
  32. package/dist/ingest-books.js.map +1 -1
  33. package/dist/ingest-images.js +3 -4
  34. package/dist/ingest-images.js.map +1 -1
  35. package/dist/ingest-notes.js +10 -24
  36. package/dist/ingest-notes.js.map +1 -1
  37. package/dist/lib/glob-paths.d.ts +6 -0
  38. package/dist/lib/glob-paths.d.ts.map +1 -0
  39. package/dist/lib/glob-paths.js +9 -0
  40. package/dist/lib/glob-paths.js.map +1 -0
  41. package/dist/lib/glob-paths.test.d.ts +2 -0
  42. package/dist/lib/glob-paths.test.d.ts.map +1 -0
  43. package/dist/lib/glob-paths.test.js +30 -0
  44. package/dist/lib/glob-paths.test.js.map +1 -0
  45. package/dist/lib/html-charset.d.ts +4 -0
  46. package/dist/lib/html-charset.d.ts.map +1 -0
  47. package/dist/lib/html-charset.js +33 -0
  48. package/dist/lib/html-charset.js.map +1 -0
  49. package/dist/lib/html-charset.test.d.ts +2 -0
  50. package/dist/lib/html-charset.test.d.ts.map +1 -0
  51. package/dist/lib/html-charset.test.js +27 -0
  52. package/dist/lib/html-charset.test.js.map +1 -0
  53. package/dist/rag/ingest.js +4 -4
  54. package/dist/rag/ingest.js.map +1 -1
  55. package/dist/reset-articles.js +1 -2
  56. package/dist/reset-articles.js.map +1 -1
  57. package/dist/reset-books.js +1 -1
  58. package/dist/reset-books.js.map +1 -1
  59. package/dist/reset-images.js +1 -1
  60. package/dist/reset-images.js.map +1 -1
  61. package/dist/reset-notes.js +1 -1
  62. package/dist/reset-notes.js.map +1 -1
  63. package/dist/revert-articles.js +1 -2
  64. package/dist/revert-articles.js.map +1 -1
  65. package/dist/revert-books.js +1 -1
  66. package/dist/revert-books.js.map +1 -1
  67. package/dist/revert-images.js +1 -1
  68. package/dist/revert-images.js.map +1 -1
  69. package/dist/revert-notes.js +1 -1
  70. package/dist/revert-notes.js.map +1 -1
  71. package/dist/s-content-format-runner.d.ts +10 -0
  72. package/dist/s-content-format-runner.d.ts.map +1 -0
  73. package/dist/s-content-format-runner.js +76 -0
  74. package/dist/s-content-format-runner.js.map +1 -0
  75. package/dist/s-content-format.js +15 -42
  76. package/dist/s-content-format.js.map +1 -1
  77. package/dist/s-content-format.test.d.ts +2 -0
  78. package/dist/s-content-format.test.d.ts.map +1 -0
  79. package/dist/s-content-format.test.js +99 -0
  80. package/dist/s-content-format.test.js.map +1 -0
  81. package/dist/update-json-articles.js +4 -37
  82. package/dist/update-json-articles.js.map +1 -1
  83. package/dist/update-raw-articles.js +2 -34
  84. package/dist/update-raw-articles.js.map +1 -1
  85. package/package.json +12 -12
  86. package/configs/dprint.json +0 -22
  87. package/dist/image-processing-photon.test.d.ts +0 -2
  88. package/dist/image-processing-photon.test.d.ts.map +0 -1
  89. package/dist/image-processing-photon.test.js +0 -27
  90. package/dist/image-processing-photon.test.js.map +0 -1
@@ -1,40 +1,10 @@
1
1
  #!/usr/bin/env node
2
2
  import { createPushoverService } from "@s-hirano-ist/s-notification";
3
3
  import * as cheerio from "cheerio";
4
- import iconv from "iconv-lite";
5
4
  import { readdirSync, readFileSync, writeFileSync } from "node:fs";
6
5
  import { join } from "node:path";
7
- function normalizeCharset(charset) {
8
- const normalized = charset.toLowerCase().replaceAll(/[^a-z0-9]/gu, "");
9
- const mapping = {
10
- shiftjis: "Shift_JIS",
11
- sjis: "Shift_JIS",
12
- xsjis: "Shift_JIS",
13
- eucjp: "EUC-JP",
14
- xeucjp: "EUC-JP",
15
- };
16
- return mapping[normalized] || charset;
17
- }
18
- function detectCharset(headers, buffer) {
19
- // 1. Content-Type ヘッダーから検出
20
- const contentType = headers.get("content-type") || "";
21
- const headerMatch = /charset=([^\s;]+)/iu.exec(contentType);
22
- if (headerMatch)
23
- return normalizeCharset(headerMatch[1]);
24
- // 2. HTML meta タグから検出(ASCII範囲で仮デコード)
25
- const preview = buffer
26
- .subarray(0, Math.min(4096, buffer.length))
27
- .toString("ascii");
28
- // <meta charset="...">
29
- const metaCharset = /<meta\s{1,200}charset=["']?([^"'\s>]+)/iu.exec(preview);
30
- if (metaCharset)
31
- return normalizeCharset(metaCharset[1]);
32
- // <meta http-equiv="Content-Type" content="...; charset=...">
33
- const metaHttpEquiv = /<meta[^>]{1,500}http-equiv=["']?Content-Type["']?[^>]{1,500}content=["'][^"']{0,500}charset=([^"'\s;]+)/iu.exec(preview);
34
- if (metaHttpEquiv)
35
- return normalizeCharset(metaHttpEquiv[1]);
36
- return "utf-8";
37
- }
6
+ import { setTimeout } from "node:timers/promises";
7
+ import { decodeHtml } from "./lib/html-charset.js";
38
8
  async function getOgTags(url) {
39
9
  try {
40
10
  console.log(`Fetching OG tags for: ${url}`);
@@ -47,8 +17,7 @@ async function getOgTags(url) {
47
17
  throw new Error(`HTTP error! status: ${response.status}`);
48
18
  }
49
19
  const buffer = Buffer.from(await response.arrayBuffer());
50
- const charset = detectCharset(response.headers, buffer);
51
- const html = iconv.decode(buffer, charset);
20
+ const html = decodeHtml(response.headers, buffer);
52
21
  const $ = cheerio.load(html);
53
22
  const ogTags = {};
54
23
  $("meta[property]").each((_, el) => {
@@ -93,9 +62,7 @@ async function processArticleFile(filePath) {
93
62
  else
94
63
  console.warn(`No OG description found for ${item.url}`);
95
64
  // Add a small delay to avoid overwhelming servers
96
- await new Promise((resolve) => {
97
- setTimeout(resolve, 1000);
98
- });
65
+ await setTimeout(1000);
99
66
  }
100
67
  // Write back to file
101
68
  writeFileSync(filePath, `${JSON.stringify(articleData, null, "\t")}\n`, "utf-8");
@@ -1 +1 @@
1
- {"version":3,"file":"update-json-articles.js","sourceRoot":"","sources":["../src/update-json-articles.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AACrE,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACnE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAkBjC,SAAS,gBAAgB,CAAC,OAAe;IACxC,MAAM,UAAU,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;IACvE,MAAM,OAAO,GAA2B;QACvC,QAAQ,EAAE,WAAW;QACrB,IAAI,EAAE,WAAW;QACjB,KAAK,EAAE,WAAW;QAClB,KAAK,EAAE,QAAQ;QACf,MAAM,EAAE,QAAQ;KAChB,CAAC;IACF,OAAO,OAAO,CAAC,UAAU,CAAC,IAAI,OAAO,CAAC;AACvC,CAAC;AAED,SAAS,aAAa,CAAC,OAAgB,EAAE,MAAc;IACtD,2BAA2B;IAC3B,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;IACtD,MAAM,WAAW,GAAG,qBAAqB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAC5D,IAAI,WAAW;QAAE,OAAO,gBAAgB,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;IAEzD,qCAAqC;IACrC,MAAM,OAAO,GAAG,MAAM;SACpB,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;SAC1C,QAAQ,CAAC,OAAO,CAAC,CAAC;IAEpB,uBAAuB;IACvB,MAAM,WAAW,GAAG,0CAA0C,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC7E,IAAI,WAAW;QAAE,OAAO,gBAAgB,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;IAEzD,8DAA8D;IAC9D,MAAM,aAAa,GAClB,2GAA2G,CAAC,IAAI,CAC/G,OAAO,CACP,CAAC;IACH,IAAI,aAAa;QAAE,OAAO,gBAAgB,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;IAE7D,OAAO,OAAO,CAAC;AAChB,CAAC;AAED,KAAK,UAAU,SAAS,CACvB,GAAW;IAEX,IAAI,CAAC;QACJ,OAAO,CAAC,GAAG,CAAC,yBAAyB,GAAG,EAAE,CAAC,CAAC;QAE5C,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YACjC,OAAO,EAAE;gBACR,YAAY,EACX,2HAA2H;aAC5H;SACD,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CAAC,uBAAuB,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QAC3D,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;QACzD,MAAM,OAAO,GAAG,aAAa,CAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACxD,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAE3C,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,MAAM,MAAM,GAA2B,EAAE,CAAC;QAC1C,CAAC,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;YAClC,MAAM,QAAQ,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,CAAC;YAChD,MAAM,OAAO,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACtC,IAAI,QAAQ,IAAI,OAAO,EAAE,CAAC;gBACzB,MAAM,CAAC,QAAQ,CAAC,GAAG,OAAO,CAAC;YAC5B,CAAC;QACF,CAAC,CAAC,CAAC;QAEH,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;QACtC,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;QACnC,MAAM,aAAa,GAAG,MAAM,CAAC,gBAAgB,CAAC,CAAC;QAE/C,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC;IAC/C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,OAAO,CAAC,KAAK,CAAC,8BAA8B,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;QAC3D,OAAO,EAAE,CAAC;IACX,CAAC;AACF,CAAC;AAED,KAAK,UAAU,kBAAkB,CAAC,QAAgB;IACjD,IAAI,CAAC;QACJ,MAAM,WAAW,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACpD,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAiB,CAAC;QAE5D,OAAO,CAAC,GAAG,CAAC,cAAc,QAAQ,KAAK,CAAC,CAAC;QAEzC,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,IAAI,EAAE,CAAC;YACrC,gCAAgC;YAChC,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;gBACxE,SAAS;YACV,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAEzC,0CAA0C;YAC1C,IAAI,MAAM,CAAC,UAAU;gBAAE,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;;gBACtD,OAAO,CAAC,IAAI,CAAC,yBAAyB,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;YAEvD,IAAI,MAAM,CAAC,OAAO;gBAAE,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;;gBAC7C,OAAO,CAAC,IAAI,CAAC,yBAAyB,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;YAEvD,IAAI,MAAM,CAAC,aAAa;gBAAE,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC;;gBAC/D,OAAO,CAAC,IAAI,CAAC,+BAA+B,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;YAE7D,kDAAkD;YAClD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;gBAC7B,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;YAC3B,CAAC,CAAC,CAAC;QACJ,CAAC;QAED,qBAAqB;QACrB,aAAa,CACZ,QAAQ,EACR,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAC9C,OAAO,CACP,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,wBAAwB,QAAQ,EAAE,CAAC,CAAC;IACjD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,OAAO,CAAC,KAAK,CAAC,oBAAoB,QAAQ,GAAG,EAAE,KAAK,CAAC,CAAC;IACvD,CAAC;AACF,CAAC;AAED,KAAK,UAAU,IAAI;IAClB,MAAM,GAAG,GAAG;QACX,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY;QACtC,iBAAiB,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB;QAChD,kBAAkB,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB;KACzC,CAAC;IAEX,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IAChE,CAAC;IAED,MAAM,mBAAmB,GAAG,qBAAqB,CAAC;QACjD,GAAG,EAAE,GAAG,CAAC,YAAY,IAAI,EAAE;QAC3B,OAAO,EAAE,GAAG,CAAC,iBAAiB,IAAI,EAAE;QACpC,QAAQ,EAAE,GAAG,CAAC,kBAAkB,IAAI,EAAE;KACtC,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;IAE1D,IAAI,CAAC;QACJ,MAAM,KAAK,GAAG,WAAW,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CACrD,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CACtB,CAAC;QAEF,OAAO,CAAC,GAAG,CAAC,SAAS,KAAK,CAAC,MAAM,wBAAwB,CAAC,CAAC;QAE3D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YAC1B,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;YACxC,MAAM,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QACpC,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAC;QACjD,MAAM,mBAAmB,CAAC,UAAU,CAAC,gCAAgC,EAAE;YACtE,MAAM,EAAE,sBAAsB;SAC9B,CAAC,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,OAAO,CAAC,KAAK,CAAC,wBAAwB,EAAE,KAAK,CAAC,CAAC;QAC/C,MAAM,mBAAmB,CAAC,WAAW,CACpC,gCAAgC,MAAM,CAAC,KAAK,CAAC,EAAE,EAC/C;YACC,MAAM,EAAE,sBAAsB;SAC9B,CACD,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;AACF,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;IAC/B,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC,CAAC,CAAC"}
1
+ {"version":3,"file":"update-json-articles.js","sourceRoot":"","sources":["../src/update-json-articles.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AACrE,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACnE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAkBnD,KAAK,UAAU,SAAS,CACvB,GAAW;IAEX,IAAI,CAAC;QACJ,OAAO,CAAC,GAAG,CAAC,yBAAyB,GAAG,EAAE,CAAC,CAAC;QAE5C,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YACjC,OAAO,EAAE;gBACR,YAAY,EACX,2HAA2H;aAC5H;SACD,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CAAC,uBAAuB,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QAC3D,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;QACzD,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAElD,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,MAAM,MAAM,GAA2B,EAAE,CAAC;QAC1C,CAAC,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;YAClC,MAAM,QAAQ,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,CAAC;YAChD,MAAM,OAAO,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACtC,IAAI,QAAQ,IAAI,OAAO,EAAE,CAAC;gBACzB,MAAM,CAAC,QAAQ,CAAC,GAAG,OAAO,CAAC;YAC5B,CAAC;QACF,CAAC,CAAC,CAAC;QAEH,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;QACtC,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;QACnC,MAAM,aAAa,GAAG,MAAM,CAAC,gBAAgB,CAAC,CAAC;QAE/C,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC;IAC/C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,OAAO,CAAC,KAAK,CAAC,8BAA8B,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;QAC3D,OAAO,EAAE,CAAC;IACX,CAAC;AACF,CAAC;AAED,KAAK,UAAU,kBAAkB,CAAC,QAAgB;IACjD,IAAI,CAAC;QACJ,MAAM,WAAW,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACpD,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAiB,CAAC;QAE5D,OAAO,CAAC,GAAG,CAAC,cAAc,QAAQ,KAAK,CAAC,CAAC;QAEzC,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,IAAI,EAAE,CAAC;YACrC,gCAAgC;YAChC,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;gBACxE,SAAS;YACV,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAEzC,0CAA0C;YAC1C,IAAI,MAAM,CAAC,UAAU;gBAAE,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;;gBACtD,OAAO,CAAC,IAAI,CAAC,yBAAyB,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;YAEvD,IAAI,MAAM,CAAC,OAAO;gBAAE,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;;gBAC7C,OAAO,CAAC,IAAI,CAAC,yBAAyB,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;YAEvD,IAAI,MAAM,CAAC,aAAa;gBAAE,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC;;gBAC/D,OAAO,CAAC,IAAI,CAAC,+BAA+B,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;YAE7D,kDAAkD;YAClD,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC;QACxB,CAAC;QAED,qBAAqB;QACrB,aAAa,CACZ,QAAQ,EACR,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAC9C,OAAO,CACP,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,wBAAwB,QAAQ,EAAE,CAAC,CAAC;IACjD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,OAAO,CAAC,KAAK,CAAC,oBAAoB,QAAQ,GAAG,EAAE,KAAK,CAAC,CAAC;IACvD,CAAC;AACF,CAAC;AAED,KAAK,UAAU,IAAI;IAClB,MAAM,GAAG,GAAG;QACX,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY;QACtC,iBAAiB,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB;QAChD,kBAAkB,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB;KACzC,CAAC;IAEX,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IAChE,CAAC;IAED,MAAM,mBAAmB,GAAG,qBAAqB,CAAC;QACjD,GAAG,EAAE,GAAG,CAAC,YAAY,IAAI,EAAE;QAC3B,OAAO,EAAE,GAAG,CAAC,iBAAiB,IAAI,EAAE;QACpC,QAAQ,EAAE,GAAG,CAAC,kBAAkB,IAAI,EAAE;KACtC,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;IAE1D,IAAI,CAAC;QACJ,MAAM,KAAK,GAAG,WAAW,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CACrD,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CACtB,CAAC;QAEF,OAAO,CAAC,GAAG,CAAC,SAAS,KAAK,CAAC,MAAM,wBAAwB,CAAC,CAAC;QAE3D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YAC1B,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;YACxC,MAAM,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QACpC,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAC;QACjD,MAAM,mBAAmB,CAAC,UAAU,CAAC,gCAAgC,EAAE;YACtE,MAAM,EAAE,sBAAsB;SAC9B,CAAC,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,OAAO,CAAC,KAAK,CAAC,wBAAwB,EAAE,KAAK,CAAC,CAAC;QAC/C,MAAM,mBAAmB,CAAC,WAAW,CACpC,gCAAgC,MAAM,CAAC,KAAK,CAAC,EAAE,EAC/C;YACC,MAAM,EAAE,sBAAsB;SAC9B,CACD,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;AACF,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;IAC/B,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC,CAAC,CAAC"}
@@ -1,43 +1,12 @@
1
1
  #!/usr/bin/env node
2
2
  import { createPushoverService } from "@s-hirano-ist/s-notification";
3
- import iconv from "iconv-lite";
4
3
  import { existsSync } from "node:fs";
5
4
  import { mkdir, readdir, readFile, writeFile } from "node:fs/promises";
6
5
  import path from "node:path";
7
6
  import TurndownService from "turndown";
7
+ import { decodeHtml } from "./lib/html-charset.js";
8
8
  const JSON_DIR = "json/article";
9
9
  const OUTPUT_DIR = "raw/article";
10
- function normalizeCharset(charset) {
11
- const normalized = charset.toLowerCase().replaceAll(/[^a-z0-9]/gu, "");
12
- const mapping = {
13
- shiftjis: "Shift_JIS",
14
- sjis: "Shift_JIS",
15
- xsjis: "Shift_JIS",
16
- eucjp: "EUC-JP",
17
- xeucjp: "EUC-JP",
18
- };
19
- return mapping[normalized] || charset;
20
- }
21
- function detectCharset(headers, buffer) {
22
- // 1. Content-Type ヘッダーから検出
23
- const contentType = headers.get("content-type") || "";
24
- const headerMatch = /charset=([^\s;]+)/iu.exec(contentType);
25
- if (headerMatch)
26
- return normalizeCharset(headerMatch[1]);
27
- // 2. HTML meta タグから検出(ASCII範囲で仮デコード)
28
- const preview = buffer
29
- .subarray(0, Math.min(4096, buffer.length))
30
- .toString("ascii");
31
- // <meta charset="...">
32
- const metaCharset = /<meta\s{1,200}charset=["']?([^"'\s>]+)/iu.exec(preview);
33
- if (metaCharset)
34
- return normalizeCharset(metaCharset[1]);
35
- // <meta http-equiv="Content-Type" content="...; charset=...">
36
- const metaHttpEquiv = /<meta[^>]{1,500}http-equiv=["']?Content-Type["']?[^>]{1,500}content=["'][^"']{0,500}charset=([^"'\s;]+)/iu.exec(preview);
37
- if (metaHttpEquiv)
38
- return normalizeCharset(metaHttpEquiv[1]);
39
- return "utf-8";
40
- }
41
10
  async function fetchWebsiteMarkdown(url) {
42
11
  try {
43
12
  const response = await fetch(url, {
@@ -49,8 +18,7 @@ async function fetchWebsiteMarkdown(url) {
49
18
  throw new Error(`HTTP error! status: ${response.status}`);
50
19
  }
51
20
  const buffer = Buffer.from(await response.arrayBuffer());
52
- const charset = detectCharset(response.headers, buffer);
53
- const html = iconv.decode(buffer, charset);
21
+ const html = decodeHtml(response.headers, buffer);
54
22
  const turndownService = new TurndownService({
55
23
  headingStyle: "atx",
56
24
  codeBlockStyle: "fenced",
@@ -1 +1 @@
1
- {"version":3,"file":"update-raw-articles.js","sourceRoot":"","sources":["../src/update-raw-articles.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AACrE,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACvE,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,eAAe,MAAM,UAAU,CAAC;AAEvC,MAAM,QAAQ,GAAG,cAAc,CAAC;AAChC,MAAM,UAAU,GAAG,aAAa,CAAC;AAEjC,SAAS,gBAAgB,CAAC,OAAe;IACxC,MAAM,UAAU,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;IACvE,MAAM,OAAO,GAA2B;QACvC,QAAQ,EAAE,WAAW;QACrB,IAAI,EAAE,WAAW;QACjB,KAAK,EAAE,WAAW;QAClB,KAAK,EAAE,QAAQ;QACf,MAAM,EAAE,QAAQ;KAChB,CAAC;IACF,OAAO,OAAO,CAAC,UAAU,CAAC,IAAI,OAAO,CAAC;AACvC,CAAC;AAED,SAAS,aAAa,CAAC,OAAgB,EAAE,MAAc;IACtD,2BAA2B;IAC3B,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;IACtD,MAAM,WAAW,GAAG,qBAAqB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAC5D,IAAI,WAAW;QAAE,OAAO,gBAAgB,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;IAEzD,qCAAqC;IACrC,MAAM,OAAO,GAAG,MAAM;SACpB,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;SAC1C,QAAQ,CAAC,OAAO,CAAC,CAAC;IAEpB,uBAAuB;IACvB,MAAM,WAAW,GAAG,0CAA0C,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC7E,IAAI,WAAW;QAAE,OAAO,gBAAgB,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;IAEzD,8DAA8D;IAC9D,MAAM,aAAa,GAClB,2GAA2G,CAAC,IAAI,CAC/G,OAAO,CACP,CAAC;IACH,IAAI,aAAa;QAAE,OAAO,gBAAgB,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;IAE7D,OAAO,OAAO,CAAC;AAChB,CAAC;AAED,KAAK,UAAU,oBAAoB,CAAC,GAAW;IAC9C,IAAI,CAAC;QACJ,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YACjC,OAAO,EAAE;gBACR,YAAY,EACX,2HAA2H;aAC5H;SACD,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CAAC,uBAAuB,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QAC3D,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;QACzD,MAAM,OAAO,GAAG,aAAa,CAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACxD,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAE3C,MAAM,eAAe,GAAG,IAAI,eAAe,CAAC;YAC3C,YAAY,EAAE,KAAK;YACnB,cAAc,EAAE,QAAQ;YACxB,gBAAgB,EAAE,GAAG;YACrB,eAAe,EAAE,IAAI;YACrB,WAAW,EAAE,GAAG;SAChB,CAAC,CAAC;QAEH,eAAe,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;QAEtE,eAAe,CAAC,OAAO,CAAC,eAAe,EAAE;YACxC,MAAM,EAAE,CAAC,GAAG,CAAC;YACb,WAAW,EAAE,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE;gBAC9B,MAAM,OAAO,GAAG,IAAyB,CAAC;gBAC1C,MAAM,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;gBAC1C,IAAI,CAAC,IAAI;oBAAE,OAAO,OAAO,CAAC;gBAE1B,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;oBACzD,OAAO,IAAI,OAAO,KAAK,IAAI,GAAG,CAAC;gBAChC,CAAC;gBACD,IAAI,CAAC;oBACJ,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC;oBAC5C,OAAO,IAAI,OAAO,KAAK,WAAW,GAAG,CAAC;gBACvC,CAAC;gBAAC,MAAM,CAAC;oBACR,OAAO,OAAO,CAAC;gBAChB,CAAC;YACF,CAAC;SACD,CAAC,CAAC;QAEH,eAAe,CAAC,OAAO,CAAC,gBAAgB,EAAE;YACzC,MAAM,EAAE,CAAC,KAAK,CAAC;YACf,WAAW,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE;gBAC/B,MAAM,OAAO,GAAG,IAAwB,CAAC;gBACzC,MAAM,GAAG,GAAG,OAAO,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;gBACxC,MAAM,GAAG,GAAG,OAAO,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;gBAC9C,IAAI,CAAC,GAAG;oBAAE,OAAO,EAAE,CAAC;gBAEpB,IAAI,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;oBACvD,OAAO,KAAK,GAAG,KAAK,GAAG,GAAG,CAAC;gBAC5B,CAAC;gBACD,IAAI,CAAC;oBACJ,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC;oBAC3C,OAAO,KAAK,GAAG,KAAK,WAAW,GAAG,CAAC;gBACpC,CAAC;gBAAC,MAAM,CAAC;oBACR,OAAO,EAAE,CAAC;gBACX,CAAC;YACF,CAAC;SACD,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAChD,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;IACxB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,OAAO,CAAC,KAAK,CAAC,mBAAmB,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;QAChD,OAAO,EAAE,CAAC;IACX,CAAC;AACF,CAAC;AAeD,KAAK,UAAU,cAAc,CAAC,QAAgB;IAC7C,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAClD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAiB,CAAC;IAEjD,sJAAsJ;IACtJ,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;IAEjC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC7B,MAAM,KAAK,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9C,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC7B,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC;QACzC,IAAI,CAAC;YACJ,MAAM,WAAW,GAAG,GAAG,CAAC,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC;YACrD,MAAM,OAAO,GAAG,kBAAkB,CAAC,WAAW,CAAC,CAAC;YAChD,MAAM,cAAc,GAAG,GAAG,OAAO,KAAK,CAAC;YACvC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;YAEzD,IAAI,UAAU,CAAC,UAAU,CAAC,IAAI,IAAI,EAAE,CAAC;gBACpC,SAAS;YACV,CAAC;YAED,MAAM,WAAW,GAAG,MAAM,oBAAoB,CAAC,GAAG,CAAC,CAAC;YAEpD,IAAI,CAAC,WAAW,EAAE,CAAC;gBAClB,SAAS;YACV,CAAC;YAED,MAAM,eAAe,GAAG,MAAM,KAAK,KAAK,GAAG;;;;;;EAM5C,KAAK;;;;EAIL,WAAW;CACZ,CAAC;YAEC,MAAM,SAAS,CAAC,UAAU,EAAE,eAAe,EAAE,OAAO,CAAC,CAAC;YACtD,OAAO,CAAC,GAAG,CAAC,aAAa,UAAU,EAAE,CAAC,CAAC;QACxC,CAAC;QAAC,MAAM,CAAC;YACR,OAAO,CAAC,KAAK,CAAC,gBAAgB,EAAE,GAAG,CAAC,CAAC;QACtC,CAAC;IACF,CAAC;AACF,CAAC;AAED,KAAK,UAAU,IAAI;IAClB,MAAM,GAAG,GAAG;QACX,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY;QACtC,iBAAiB,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB;QAChD,kBAAkB,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB;KACzC,CAAC;IAEX,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IAChE,CAAC;IAED,MAAM,mBAAmB,GAAG,qBAAqB,CAAC;QACjD,GAAG,EAAE,GAAG,CAAC,YAAY,IAAI,EAAE;QAC3B,OAAO,EAAE,GAAG,CAAC,iBAAiB,IAAI,EAAE;QACpC,QAAQ,EAAE,GAAG,CAAC,kBAAkB,IAAI,EAAE;KACtC,CAAC,CAAC;IAEH,IAAI,CAAC;QACJ,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC1C,MAAM,aAAa,GAAG,SAAS;aAC7B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;aACxC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC;QAE3C,KAAK,MAAM,QAAQ,IAAI,aAAa,EAAE,CAAC;YACtC,OAAO,CAAC,GAAG,CAAC,eAAe,QAAQ,EAAE,CAAC,CAAC;YACvC,MAAM,cAAc,CAAC,QAAQ,CAAC,CAAC;QAChC,CAAC;QACD,MAAM,mBAAmB,CAAC,UAAU,CAAC,+BAA+B,EAAE;YACrE,MAAM,EAAE,qBAAqB;SAC7B,CAAC,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,OAAO,CAAC,KAAK,CAAC,wBAAwB,EAAE,KAAK,CAAC,CAAC;QAC/C,MAAM,mBAAmB,CAAC,WAAW,CACpC,+BAA+B,MAAM,CAAC,KAAK,CAAC,EAAE,EAC9C;YACC,MAAM,EAAE,qBAAqB;SAC7B,CACD,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;AACF,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;IAC/B,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC,CAAC,CAAC"}
1
+ {"version":3,"file":"update-raw-articles.js","sourceRoot":"","sources":["../src/update-raw-articles.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AACrE,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACvE,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,eAAe,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAEnD,MAAM,QAAQ,GAAG,cAAc,CAAC;AAChC,MAAM,UAAU,GAAG,aAAa,CAAC;AAEjC,KAAK,UAAU,oBAAoB,CAAC,GAAW;IAC9C,IAAI,CAAC;QACJ,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YACjC,OAAO,EAAE;gBACR,YAAY,EACX,2HAA2H;aAC5H;SACD,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CAAC,uBAAuB,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QAC3D,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;QACzD,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAElD,MAAM,eAAe,GAAG,IAAI,eAAe,CAAC;YAC3C,YAAY,EAAE,KAAK;YACnB,cAAc,EAAE,QAAQ;YACxB,gBAAgB,EAAE,GAAG;YACrB,eAAe,EAAE,IAAI;YACrB,WAAW,EAAE,GAAG;SAChB,CAAC,CAAC;QAEH,eAAe,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;QAEtE,eAAe,CAAC,OAAO,CAAC,eAAe,EAAE;YACxC,MAAM,EAAE,CAAC,GAAG,CAAC;YACb,WAAW,EAAE,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE;gBAC9B,MAAM,OAAO,GAAG,IAAyB,CAAC;gBAC1C,MAAM,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;gBAC1C,IAAI,CAAC,IAAI;oBAAE,OAAO,OAAO,CAAC;gBAE1B,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;oBACzD,OAAO,IAAI,OAAO,KAAK,IAAI,GAAG,CAAC;gBAChC,CAAC;gBACD,IAAI,CAAC;oBACJ,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC;oBAC5C,OAAO,IAAI,OAAO,KAAK,WAAW,GAAG,CAAC;gBACvC,CAAC;gBAAC,MAAM,CAAC;oBACR,OAAO,OAAO,CAAC;gBAChB,CAAC;YACF,CAAC;SACD,CAAC,CAAC;QAEH,eAAe,CAAC,OAAO,CAAC,gBAAgB,EAAE;YACzC,MAAM,EAAE,CAAC,KAAK,CAAC;YACf,WAAW,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE;gBAC/B,MAAM,OAAO,GAAG,IAAwB,CAAC;gBACzC,MAAM,GAAG,GAAG,OAAO,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;gBACxC,MAAM,GAAG,GAAG,OAAO,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;gBAC9C,IAAI,CAAC,GAAG;oBAAE,OAAO,EAAE,CAAC;gBAEpB,IAAI,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;oBACvD,OAAO,KAAK,GAAG,KAAK,GAAG,GAAG,CAAC;gBAC5B,CAAC;gBACD,IAAI,CAAC;oBACJ,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC;oBAC3C,OAAO,KAAK,GAAG,KAAK,WAAW,GAAG,CAAC;gBACpC,CAAC;gBAAC,MAAM,CAAC;oBACR,OAAO,EAAE,CAAC;gBACX,CAAC;YACF,CAAC;SACD,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAChD,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;IACxB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,OAAO,CAAC,KAAK,CAAC,mBAAmB,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;QAChD,OAAO,EAAE,CAAC;IACX,CAAC;AACF,CAAC;AAeD,KAAK,UAAU,cAAc,CAAC,QAAgB;IAC7C,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAClD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAiB,CAAC;IAEjD,sJAAsJ;IACtJ,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;IAEjC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC7B,MAAM,KAAK,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9C,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC7B,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC;QACzC,IAAI,CAAC;YACJ,MAAM,WAAW,GAAG,GAAG,CAAC,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC;YACrD,MAAM,OAAO,GAAG,kBAAkB,CAAC,WAAW,CAAC,CAAC;YAChD,MAAM,cAAc,GAAG,GAAG,OAAO,KAAK,CAAC;YACvC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;YAEzD,IAAI,UAAU,CAAC,UAAU,CAAC,IAAI,IAAI,EAAE,CAAC;gBACpC,SAAS;YACV,CAAC;YAED,MAAM,WAAW,GAAG,MAAM,oBAAoB,CAAC,GAAG,CAAC,CAAC;YAEpD,IAAI,CAAC,WAAW,EAAE,CAAC;gBAClB,SAAS;YACV,CAAC;YAED,MAAM,eAAe,GAAG,MAAM,KAAK,KAAK,GAAG;;;;;;EAM5C,KAAK;;;;EAIL,WAAW;CACZ,CAAC;YAEC,MAAM,SAAS,CAAC,UAAU,EAAE,eAAe,EAAE,OAAO,CAAC,CAAC;YACtD,OAAO,CAAC,GAAG,CAAC,aAAa,UAAU,EAAE,CAAC,CAAC;QACxC,CAAC;QAAC,MAAM,CAAC;YACR,OAAO,CAAC,KAAK,CAAC,gBAAgB,EAAE,GAAG,CAAC,CAAC;QACtC,CAAC;IACF,CAAC;AACF,CAAC;AAED,KAAK,UAAU,IAAI;IAClB,MAAM,GAAG,GAAG;QACX,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY;QACtC,iBAAiB,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB;QAChD,kBAAkB,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB;KACzC,CAAC;IAEX,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IAChE,CAAC;IAED,MAAM,mBAAmB,GAAG,qBAAqB,CAAC;QACjD,GAAG,EAAE,GAAG,CAAC,YAAY,IAAI,EAAE;QAC3B,OAAO,EAAE,GAAG,CAAC,iBAAiB,IAAI,EAAE;QACpC,QAAQ,EAAE,GAAG,CAAC,kBAAkB,IAAI,EAAE;KACtC,CAAC,CAAC;IAEH,IAAI,CAAC;QACJ,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC1C,MAAM,aAAa,GAAG,SAAS;aAC7B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;aACxC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC;QAE3C,KAAK,MAAM,QAAQ,IAAI,aAAa,EAAE,CAAC;YACtC,OAAO,CAAC,GAAG,CAAC,eAAe,QAAQ,EAAE,CAAC,CAAC;YACvC,MAAM,cAAc,CAAC,QAAQ,CAAC,CAAC;QAChC,CAAC;QACD,MAAM,mBAAmB,CAAC,UAAU,CAAC,+BAA+B,EAAE;YACrE,MAAM,EAAE,qBAAqB;SAC7B,CAAC,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,OAAO,CAAC,KAAK,CAAC,wBAAwB,EAAE,KAAK,CAAC,CAAC;QAC/C,MAAM,mBAAmB,CAAC,WAAW,CACpC,+BAA+B,MAAM,CAAC,KAAK,CAAC,EAAE,EAC9C;YACC,MAAM,EAAE,qBAAqB;SAC7B,CACD,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;AACF,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;IAC/B,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC,CAAC,CAAC"}
package/package.json CHANGED
@@ -1,7 +1,10 @@
1
1
  {
2
2
  "name": "@s-hirano-ist/s-scripts",
3
- "version": "1.25.2",
3
+ "version": "1.26.1",
4
4
  "description": "CLI scripts for s-private data operations",
5
+ "engines": {
6
+ "node": ">=24"
7
+ },
5
8
  "type": "module",
6
9
  "bin": {
7
10
  "s-content-format": "./dist/s-content-format.js",
@@ -49,25 +52,22 @@
49
52
  "access": "public"
50
53
  },
51
54
  "dependencies": {
52
- "@googleapis/books": "8.0.1",
55
+ "@googleapis/books": "9.0.0",
53
56
  "cheerio": "1.2.0",
54
- "dprint": "0.55.1",
55
- "glob": "13.0.6",
56
- "gray-matter": "4.0.3",
57
- "iconv-lite": "0.7.2",
58
- "js-yaml": "5.2.1",
57
+ "js-yaml": "5.3.0",
58
+ "oxfmt": "0.63.0",
59
59
  "turndown": "7.2.4",
60
- "@s-hirano-ist/s-core": "1.21.2",
61
- "@s-hirano-ist/s-database": "1.21.0",
62
60
  "@s-hirano-ist/s-image-processing": "1.0.0",
63
- "@s-hirano-ist/s-search": "1.18.13",
61
+ "@s-hirano-ist/s-database": "1.21.0",
62
+ "@s-hirano-ist/s-core": "1.21.3",
64
63
  "@s-hirano-ist/s-notification": "1.18.5",
64
+ "@s-hirano-ist/s-search": "1.18.13",
65
65
  "@s-hirano-ist/s-storage": "1.18.9"
66
66
  },
67
67
  "devDependencies": {
68
- "@types/node": "24.13.2",
68
+ "@types/node": "24.13.3",
69
69
  "@types/turndown": "5.0.6",
70
- "tsx": "4.23.0",
70
+ "tsx": "4.23.12",
71
71
  "typescript": "6.0.3"
72
72
  },
73
73
  "scripts": {
@@ -1,22 +0,0 @@
1
- {
2
- "$schema": "https://dprint.dev/schemas/v0.json",
3
- "lineWidth": 80,
4
- "indentWidth": 2,
5
- "useTabs": false,
6
- "markdown": {
7
- "lineWidth": 80,
8
- "textWrap": "maintain"
9
- },
10
- "json": {
11
- "indentWidth": 2
12
- },
13
- "yaml": {
14
- "indentWidth": 2,
15
- "quotes": "preferDouble"
16
- },
17
- "plugins": [
18
- "https://plugins.dprint.dev/markdown-0.21.1.wasm",
19
- "https://plugins.dprint.dev/json-0.21.3.wasm",
20
- "https://plugins.dprint.dev/g-plane/pretty_yaml-v0.6.0.wasm"
21
- ]
22
- }
@@ -1,2 +0,0 @@
1
- export {};
2
- //# sourceMappingURL=image-processing-photon.test.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"image-processing-photon.test.d.ts","sourceRoot":"","sources":["../src/image-processing-photon.test.ts"],"names":[],"mappings":""}
@@ -1,27 +0,0 @@
1
- import { convertToWebp, createWebpThumbnail, readImageMetadata, } from "@s-hirano-ist/s-image-processing/node";
2
- import { describe, expect, test } from "vitest";
3
- const PNG_BYTES = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAEElEQVR4AQEFAPr/AP8AAP8FAAH/+lyI0QAAAABJRU5ErkJggg==", "base64");
4
- function isWebp(bytes) {
5
- return (bytes.length >= 12 &&
6
- Buffer.from(bytes.subarray(0, 4)).toString("ascii") === "RIFF" &&
7
- Buffer.from(bytes.subarray(8, 12)).toString("ascii") === "WEBP");
8
- }
9
- describe("Photon image processing for scripts", () => {
10
- test("should read metadata and create a WebP thumbnail from fixture bytes", async () => {
11
- const metadata = await readImageMetadata(PNG_BYTES);
12
- const thumbnailBytes = await createWebpThumbnail(PNG_BYTES, {
13
- width: 192,
14
- height: 192,
15
- });
16
- expect(metadata).toEqual({ width: 1, height: 1, format: "png" });
17
- expect(isWebp(thumbnailBytes)).toBe(true);
18
- });
19
- test("should convert fixture bytes to WebP", async () => {
20
- const webpBytes = await convertToWebp(PNG_BYTES);
21
- expect(isWebp(webpBytes)).toBe(true);
22
- });
23
- test("should reject GIF fixture bytes", async () => {
24
- await expect(readImageMetadata(Buffer.from("GIF89a"))).rejects.toThrow("Unsupported image signature.");
25
- });
26
- });
27
- //# sourceMappingURL=image-processing-photon.test.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"image-processing-photon.test.js","sourceRoot":"","sources":["../src/image-processing-photon.test.ts"],"names":[],"mappings":"AAAA,OAAO,EACN,aAAa,EACb,mBAAmB,EACnB,iBAAiB,GACjB,MAAM,uCAAuC,CAAC;AAC/C,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAC;AAEhD,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAC5B,sGAAsG,EACtG,QAAQ,CACR,CAAC;AAEF,SAAS,MAAM,CAAC,KAAiB;IAChC,OAAO,CACN,KAAK,CAAC,MAAM,IAAI,EAAE;QAClB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,MAAM;QAC9D,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,MAAM,CAC/D,CAAC;AACH,CAAC;AAED,QAAQ,CAAC,qCAAqC,EAAE,GAAG,EAAE;IACpD,IAAI,CAAC,qEAAqE,EAAE,KAAK,IAAI,EAAE;QACtF,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAAC,SAAS,CAAC,CAAC;QACpD,MAAM,cAAc,GAAG,MAAM,mBAAmB,CAAC,SAAS,EAAE;YAC3D,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,GAAG;SACX,CAAC,CAAC;QAEH,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QACjE,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;QACvD,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,SAAS,CAAC,CAAC;QAEjD,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACtC,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,iCAAiC,EAAE,KAAK,IAAI,EAAE;QAClD,MAAM,MAAM,CAAC,iBAAiB,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CACrE,8BAA8B,CAC9B,CAAC;IACH,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC"}