@s-hirano-ist/s-scripts 1.26.0 → 1.26.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cleanup-minio-images.js +11 -11
- package/dist/cleanup-minio-images.js.map +1 -1
- package/dist/enrich-books.js +4 -6
- package/dist/enrich-books.js.map +1 -1
- package/dist/fetch-articles.js +1 -2
- package/dist/fetch-articles.js.map +1 -1
- package/dist/fetch-books.js +3 -5
- package/dist/fetch-books.js.map +1 -1
- package/dist/fetch-images.js +3 -5
- package/dist/fetch-images.js.map +1 -1
- package/dist/fetch-notes.js +1 -2
- package/dist/fetch-notes.js.map +1 -1
- package/dist/fix-book-frontmatter.js +2 -2
- package/dist/fix-book-frontmatter.js.map +1 -1
- package/dist/fix-image-formats.js +2 -2
- package/dist/fix-image-formats.js.map +1 -1
- package/dist/ingest-articles.js +3 -4
- package/dist/ingest-articles.js.map +1 -1
- package/dist/ingest-books.js +4 -5
- package/dist/ingest-books.js.map +1 -1
- package/dist/ingest-images.js +3 -4
- package/dist/ingest-images.js.map +1 -1
- package/dist/ingest-notes.js +3 -4
- package/dist/ingest-notes.js.map +1 -1
- package/dist/lib/glob-paths.d.ts +6 -0
- package/dist/lib/glob-paths.d.ts.map +1 -0
- package/dist/lib/glob-paths.js +9 -0
- package/dist/lib/glob-paths.js.map +1 -0
- package/dist/lib/glob-paths.test.d.ts +2 -0
- package/dist/lib/glob-paths.test.d.ts.map +1 -0
- package/dist/lib/glob-paths.test.js +30 -0
- package/dist/lib/glob-paths.test.js.map +1 -0
- package/dist/lib/html-charset.d.ts +4 -0
- package/dist/lib/html-charset.d.ts.map +1 -0
- package/dist/lib/html-charset.js +33 -0
- package/dist/lib/html-charset.js.map +1 -0
- package/dist/lib/html-charset.test.d.ts +2 -0
- package/dist/lib/html-charset.test.d.ts.map +1 -0
- package/dist/lib/html-charset.test.js +27 -0
- package/dist/lib/html-charset.test.js.map +1 -0
- package/dist/rag/ingest.js +4 -4
- package/dist/rag/ingest.js.map +1 -1
- package/dist/reset-articles.js +1 -2
- package/dist/reset-articles.js.map +1 -1
- package/dist/reset-books.js +1 -1
- package/dist/reset-books.js.map +1 -1
- package/dist/reset-images.js +1 -1
- package/dist/reset-images.js.map +1 -1
- package/dist/reset-notes.js +1 -1
- package/dist/reset-notes.js.map +1 -1
- package/dist/revert-articles.js +1 -2
- package/dist/revert-articles.js.map +1 -1
- package/dist/revert-books.js +1 -1
- package/dist/revert-books.js.map +1 -1
- package/dist/revert-images.js +1 -1
- package/dist/revert-images.js.map +1 -1
- package/dist/revert-notes.js +1 -1
- package/dist/revert-notes.js.map +1 -1
- package/dist/s-content-format-runner.d.ts +4 -1
- package/dist/s-content-format-runner.d.ts.map +1 -1
- package/dist/s-content-format-runner.js +8 -9
- package/dist/s-content-format-runner.js.map +1 -1
- package/dist/s-content-format.test.js +11 -5
- package/dist/s-content-format.test.js.map +1 -1
- package/dist/update-json-articles.js +4 -37
- package/dist/update-json-articles.js.map +1 -1
- package/dist/update-raw-articles.js +2 -34
- package/dist/update-raw-articles.js.map +1 -1
- package/package.json +8 -7
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
import { spawn } from "node:child_process";
|
|
2
2
|
import { readFileSync } from "node:fs";
|
|
3
|
-
import { createRequire } from "node:module";
|
|
4
3
|
import path from "node:path";
|
|
5
|
-
|
|
4
|
+
import { fileURLToPath } from "node:url";
|
|
6
5
|
const CONTENT_EXTENSIONS = [
|
|
7
6
|
".md",
|
|
8
7
|
".markdown",
|
|
@@ -25,7 +24,7 @@ const DEFAULT_EXCLUDES = [
|
|
|
25
24
|
"!**/.env.*",
|
|
26
25
|
];
|
|
27
26
|
function resolveOxfmtBin() {
|
|
28
|
-
const packageJsonPath =
|
|
27
|
+
const packageJsonPath = fileURLToPath(import.meta.resolve("oxfmt/package.json"));
|
|
29
28
|
const manifest = JSON.parse(readFileSync(packageJsonPath, "utf8"));
|
|
30
29
|
const bin = typeof manifest.bin === "string" ? manifest.bin : manifest.bin?.oxfmt;
|
|
31
30
|
if (bin === undefined) {
|
|
@@ -67,11 +66,11 @@ export function buildOxfmtArgs(rawArgs) {
|
|
|
67
66
|
...DEFAULT_EXCLUDES,
|
|
68
67
|
];
|
|
69
68
|
}
|
|
70
|
-
export function runContentFormat(rawArgs, cwd = process.cwd()) {
|
|
71
|
-
const child = spawn(process.execPath, [resolveOxfmtBin(), ...buildOxfmtArgs(rawArgs)], { cwd, stdio: "inherit" });
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
69
|
+
export function runContentFormat(rawArgs, cwd = process.cwd(), options = {}) {
|
|
70
|
+
const child = spawn(process.execPath, [resolveOxfmtBin(), ...buildOxfmtArgs(rawArgs)], { cwd, stdio: options.stdio ?? "inherit" });
|
|
71
|
+
const { promise, resolve, reject } = Promise.withResolvers();
|
|
72
|
+
child.once("error", reject);
|
|
73
|
+
child.once("exit", (code, signal) => resolve({ code, signal }));
|
|
74
|
+
return promise;
|
|
76
75
|
}
|
|
77
76
|
//# sourceMappingURL=s-content-format-runner.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"s-content-format-runner.js","sourceRoot":"","sources":["../src/s-content-format-runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,
|
|
1
|
+
{"version":3,"file":"s-content-format-runner.js","sourceRoot":"","sources":["../src/s-content-format-runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,MAAM,kBAAkB,GAAG;IAC1B,KAAK;IACL,WAAW;IACX,OAAO;IACP,QAAQ;IACR,QAAQ;IACR,MAAM;IACN,OAAO;CACE,CAAC;AAEX,MAAM,gBAAgB,GAAG;IACxB,kBAAkB;IAClB,UAAU;IACV,WAAW;IACX,cAAc;IACd,oBAAoB;IACpB,cAAc;IACd,uBAAuB;IACvB,eAAe;IACf,UAAU;IACV,YAAY;CACH,CAAC;AAeX,SAAS,eAAe;IACvB,MAAM,eAAe,GAAG,aAAa,CACpC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC,CACzC,CAAC;IACF,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAC1B,YAAY,CAAC,eAAe,EAAE,MAAM,CAAC,CACb,CAAC;IAC1B,MAAM,GAAG,GACR,OAAO,QAAQ,CAAC,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC;IAEvE,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAC;IAC7E,CAAC;IAED,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,GAAG,CAAC,CAAC;AACzD,CAAC;AAED,SAAS,aAAa,CAAC,MAAc;IACpC,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;IACzC,OAAO,kBAAkB,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAC5C,WAAW,CAAC,QAAQ,CAAC,SAAS,CAAC,CAC/B,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,MAAc;IACvC,IAAI,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC;QAC3B,OAAO,CAAC,MAAM,CAAC,CAAC;IACjB,CAAC;IAED,IAAI,gBAAgB,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACpD,OAAO,gBAAgB,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACvC,gBAAgB,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAClD,CAAC;IACD,MAAM,IAAI,GAAG,gBAAgB,KAAK,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,gBAAgB,CAAC;IAC9D,OAAO,kBAAkB,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,GAAG,IAAI,QAAQ,SAAS,EAAE,CAAC,CAAC;AAC1E,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,OAA0B;IACxD,MAAM,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;IAC5C,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,KAAK,SAAS,CAAC,CAAC;IAC7D,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;IAC/D,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9D,MAAM,cAAc,GAAG,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,CAClE,CAAC,MAAM,EAAE,EAAE,CAAC,gBAAgB,CAAC,MAAM,CAAC,CACpC,CAAC;IACF,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IAC5D,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;IAEnE,OAAO;QACN,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;QAC/B,UAAU;QACV,UAAU;QACV,yBAAyB;QACzB,iCAAiC;QACjC,GAAG,OAAO;QACV,GAAG,cAAc;QACjB,GAAG,gBAAgB;KACnB,CAAC;AACH,CAAC;AAED,MAAM,UAAU,gBAAgB,CAC/B,OAA0B,EAC1B,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,EACnB,UAAgC,EAAE;IAElC,MAAM,KAAK,GAAG,KAAK,CAClB,OAAO,CAAC,QAAQ,EAChB,CAAC,eAAe,EAAE,EAAE,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC,EAC/C,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,SAAS,EAAE,CAC1C,CAAC;IAEF,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,GACjC,OAAO,CAAC,aAAa,EAAuB,CAAC;IAC9C,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAC5B,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;IAChE,OAAO,OAAO,CAAC;AAChB,CAAC"}
|
|
@@ -45,7 +45,7 @@ describe("runContentFormat", () => {
|
|
|
45
45
|
writeFile(yamlPath, "name: example\nitems:\n - one\n"),
|
|
46
46
|
writeFile(typescriptPath, "const value=1\n"),
|
|
47
47
|
]);
|
|
48
|
-
const result = await runContentFormat([], directory);
|
|
48
|
+
const result = await runContentFormat([], directory, { stdio: "ignore" });
|
|
49
49
|
expect(result).toEqual({ code: 0, signal: null });
|
|
50
50
|
expect(await readFile(markdownPath, "utf8")).not.toContain("- item");
|
|
51
51
|
expect(await readFile(jsonPath, "utf8")).not.toContain('"name":"example"');
|
|
@@ -57,11 +57,15 @@ describe("runContentFormat", () => {
|
|
|
57
57
|
const jsonPath = path.join(directory, "content.json");
|
|
58
58
|
const original = '{"name":"example","enabled":true}\n';
|
|
59
59
|
await writeFile(jsonPath, original);
|
|
60
|
-
const unformattedResult = await runContentFormat(["--check"], directory
|
|
60
|
+
const unformattedResult = await runContentFormat(["--check"], directory, {
|
|
61
|
+
stdio: "ignore",
|
|
62
|
+
});
|
|
61
63
|
expect(unformattedResult.code).not.toBe(0);
|
|
62
64
|
expect(await readFile(jsonPath, "utf8")).toBe(original);
|
|
63
|
-
expect((await runContentFormat([], directory)).code).toBe(0);
|
|
64
|
-
expect((await runContentFormat(["--check"], directory
|
|
65
|
+
expect((await runContentFormat([], directory, { stdio: "ignore" })).code).toBe(0);
|
|
66
|
+
expect((await runContentFormat(["--check"], directory, {
|
|
67
|
+
stdio: "ignore",
|
|
68
|
+
})).code).toBe(0);
|
|
65
69
|
});
|
|
66
70
|
test("keeps default exclusions and explicit directory boundaries", async () => {
|
|
67
71
|
const directory = await createTemporaryDirectory();
|
|
@@ -83,7 +87,9 @@ describe("runContentFormat", () => {
|
|
|
83
87
|
writeFile(otherPath, unformattedJson),
|
|
84
88
|
writeFile(rawPath, unformattedMarkdown),
|
|
85
89
|
]);
|
|
86
|
-
const result = await runContentFormat(["selected"], directory
|
|
90
|
+
const result = await runContentFormat(["selected"], directory, {
|
|
91
|
+
stdio: "ignore",
|
|
92
|
+
});
|
|
87
93
|
expect(result.code).toBe(0);
|
|
88
94
|
expect(await readFile(selectedPath, "utf8")).not.toBe(unformattedJson);
|
|
89
95
|
expect(await readFile(otherPath, "utf8")).toBe(unformattedJson);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"s-content-format.test.js","sourceRoot":"","sources":["../src/s-content-format.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC3E,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAC;AAC3D,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAEhF,MAAM,oBAAoB,GAAa,EAAE,CAAC;AAE1C,KAAK,UAAU,wBAAwB;IACtC,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,mBAAmB,CAAC,CAAC,CAAC;IAC1E,oBAAoB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACrC,OAAO,SAAS,CAAC;AAClB,CAAC;AAED,SAAS,CAAC,KAAK,IAAI,EAAE;IACpB,MAAM,OAAO,CAAC,GAAG,CAChB,oBAAoB;SAClB,MAAM,CAAC,CAAC,CAAC;SACT,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CACrE,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC/B,IAAI,CAAC,uDAAuD,EAAE,GAAG,EAAE;QAClE,MAAM,IAAI,GAAG,cAAc,CAAC,EAAE,CAAC,CAAC;QAEhC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;QAClC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,yBAAyB,CAAC,CAAC;QAClD,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;QACpC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;QACtC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;QACtC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACjC,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,kDAAkD,EAAE,GAAG,EAAE;QAC7D,MAAM,IAAI,GAAG,cAAc,CAAC,CAAC,SAAS,EAAE,mBAAmB,EAAE,OAAO,CAAC,CAAC,CAAC;QAEvE,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;QAClC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC;QAC5C,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;QACxC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;IACjC,IAAI,CAAC,gEAAgE,EAAE,KAAK,IAAI,EAAE;QACjF,MAAM,SAAS,GAAG,MAAM,wBAAwB,EAAE,CAAC;QACnD,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;QACxD,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;QACtD,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;QACtD,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;QACzD,MAAM,OAAO,CAAC,GAAG,CAAC;YACjB,SAAS,CAAC,YAAY,EAAE,uBAAuB,CAAC;YAChD,SAAS,CAAC,QAAQ,EAAE,qCAAqC,CAAC;YAC1D,SAAS,CAAC,QAAQ,EAAE,iCAAiC,CAAC;YACtD,SAAS,CAAC,cAAc,EAAE,mBAAmB,CAAC;SAC9C,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"s-content-format.test.js","sourceRoot":"","sources":["../src/s-content-format.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC3E,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAC;AAC3D,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAEhF,MAAM,oBAAoB,GAAa,EAAE,CAAC;AAE1C,KAAK,UAAU,wBAAwB;IACtC,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,mBAAmB,CAAC,CAAC,CAAC;IAC1E,oBAAoB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACrC,OAAO,SAAS,CAAC;AAClB,CAAC;AAED,SAAS,CAAC,KAAK,IAAI,EAAE;IACpB,MAAM,OAAO,CAAC,GAAG,CAChB,oBAAoB;SAClB,MAAM,CAAC,CAAC,CAAC;SACT,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CACrE,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC/B,IAAI,CAAC,uDAAuD,EAAE,GAAG,EAAE;QAClE,MAAM,IAAI,GAAG,cAAc,CAAC,EAAE,CAAC,CAAC;QAEhC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;QAClC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,yBAAyB,CAAC,CAAC;QAClD,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;QACpC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;QACtC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;QACtC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACjC,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,kDAAkD,EAAE,GAAG,EAAE;QAC7D,MAAM,IAAI,GAAG,cAAc,CAAC,CAAC,SAAS,EAAE,mBAAmB,EAAE,OAAO,CAAC,CAAC,CAAC;QAEvE,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;QAClC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC;QAC5C,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;QACxC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;IACjC,IAAI,CAAC,gEAAgE,EAAE,KAAK,IAAI,EAAE;QACjF,MAAM,SAAS,GAAG,MAAM,wBAAwB,EAAE,CAAC;QACnD,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;QACxD,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;QACtD,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;QACtD,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;QACzD,MAAM,OAAO,CAAC,GAAG,CAAC;YACjB,SAAS,CAAC,YAAY,EAAE,uBAAuB,CAAC;YAChD,SAAS,CAAC,QAAQ,EAAE,qCAAqC,CAAC;YAC1D,SAAS,CAAC,QAAQ,EAAE,iCAAiC,CAAC;YACtD,SAAS,CAAC,cAAc,EAAE,mBAAmB,CAAC;SAC9C,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,EAAE,EAAE,SAAS,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;QAE1E,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;QAClD,MAAM,CAAC,MAAM,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QACvE,MAAM,CAAC,MAAM,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;QAC3E,MAAM,CAAC,MAAM,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QACnE,MAAM,CAAC,MAAM,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IAC1E,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,2DAA2D,EAAE,KAAK,IAAI,EAAE;QAC5E,MAAM,SAAS,GAAG,MAAM,wBAAwB,EAAE,CAAC;QACnD,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;QACtD,MAAM,QAAQ,GAAG,qCAAqC,CAAC;QACvD,MAAM,SAAS,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAEpC,MAAM,iBAAiB,GAAG,MAAM,gBAAgB,CAAC,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE;YACxE,KAAK,EAAE,QAAQ;SACf,CAAC,CAAC;QAEH,MAAM,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC3C,MAAM,CAAC,MAAM,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAExD,MAAM,CACL,CAAC,MAAM,gBAAgB,CAAC,EAAE,EAAE,SAAS,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,IAAI,CACjE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACV,MAAM,CACL,CACC,MAAM,gBAAgB,CAAC,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE;YAC9C,KAAK,EAAE,QAAQ;SACf,CAAC,CACF,CAAC,IAAI,CACN,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACX,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;QAC7E,MAAM,SAAS,GAAG,MAAM,wBAAwB,EAAE,CAAC;QACnD,MAAM,iBAAiB,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;QAC3D,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QACrD,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;QACpE,MAAM,OAAO,CAAC,GAAG,CAAC;YACjB,KAAK,CAAC,iBAAiB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;YAC7C,KAAK,CAAC,cAAc,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;YAC1C,KAAK,CAAC,YAAY,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;SACxC,CAAC,CAAC;QACH,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,eAAe,CAAC,CAAC;QACnE,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,YAAY,CAAC,CAAC;QAC1D,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;QAClD,MAAM,eAAe,GAAG,sBAAsB,CAAC;QAC/C,MAAM,mBAAmB,GAAG,qBAAqB,CAAC;QAClD,MAAM,OAAO,CAAC,GAAG,CAAC;YACjB,SAAS,CAAC,YAAY,EAAE,eAAe,CAAC;YACxC,SAAS,CAAC,SAAS,EAAE,eAAe,CAAC;YACrC,SAAS,CAAC,OAAO,EAAE,mBAAmB,CAAC;SACvC,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,CAAC,UAAU,CAAC,EAAE,SAAS,EAAE;YAC9D,KAAK,EAAE,QAAQ;SACf,CAAC,CAAC;QAEH,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,CAAC,MAAM,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QACvE,MAAM,CAAC,MAAM,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAChE,MAAM,CAAC,MAAM,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IACnE,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC"}
|
|
@@ -1,40 +1,10 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { createPushoverService } from "@s-hirano-ist/s-notification";
|
|
3
3
|
import * as cheerio from "cheerio";
|
|
4
|
-
import iconv from "iconv-lite";
|
|
5
4
|
import { readdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
6
5
|
import { join } from "node:path";
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
const mapping = {
|
|
10
|
-
shiftjis: "Shift_JIS",
|
|
11
|
-
sjis: "Shift_JIS",
|
|
12
|
-
xsjis: "Shift_JIS",
|
|
13
|
-
eucjp: "EUC-JP",
|
|
14
|
-
xeucjp: "EUC-JP",
|
|
15
|
-
};
|
|
16
|
-
return mapping[normalized] || charset;
|
|
17
|
-
}
|
|
18
|
-
function detectCharset(headers, buffer) {
|
|
19
|
-
// 1. Content-Type ヘッダーから検出
|
|
20
|
-
const contentType = headers.get("content-type") || "";
|
|
21
|
-
const headerMatch = /charset=([^\s;]+)/iu.exec(contentType);
|
|
22
|
-
if (headerMatch)
|
|
23
|
-
return normalizeCharset(headerMatch[1]);
|
|
24
|
-
// 2. HTML meta タグから検出(ASCII範囲で仮デコード)
|
|
25
|
-
const preview = buffer
|
|
26
|
-
.subarray(0, Math.min(4096, buffer.length))
|
|
27
|
-
.toString("ascii");
|
|
28
|
-
// <meta charset="...">
|
|
29
|
-
const metaCharset = /<meta\s{1,200}charset=["']?([^"'\s>]+)/iu.exec(preview);
|
|
30
|
-
if (metaCharset)
|
|
31
|
-
return normalizeCharset(metaCharset[1]);
|
|
32
|
-
// <meta http-equiv="Content-Type" content="...; charset=...">
|
|
33
|
-
const metaHttpEquiv = /<meta[^>]{1,500}http-equiv=["']?Content-Type["']?[^>]{1,500}content=["'][^"']{0,500}charset=([^"'\s;]+)/iu.exec(preview);
|
|
34
|
-
if (metaHttpEquiv)
|
|
35
|
-
return normalizeCharset(metaHttpEquiv[1]);
|
|
36
|
-
return "utf-8";
|
|
37
|
-
}
|
|
6
|
+
import { setTimeout } from "node:timers/promises";
|
|
7
|
+
import { decodeHtml } from "./lib/html-charset.js";
|
|
38
8
|
async function getOgTags(url) {
|
|
39
9
|
try {
|
|
40
10
|
console.log(`Fetching OG tags for: ${url}`);
|
|
@@ -47,8 +17,7 @@ async function getOgTags(url) {
|
|
|
47
17
|
throw new Error(`HTTP error! status: ${response.status}`);
|
|
48
18
|
}
|
|
49
19
|
const buffer = Buffer.from(await response.arrayBuffer());
|
|
50
|
-
const
|
|
51
|
-
const html = iconv.decode(buffer, charset);
|
|
20
|
+
const html = decodeHtml(response.headers, buffer);
|
|
52
21
|
const $ = cheerio.load(html);
|
|
53
22
|
const ogTags = {};
|
|
54
23
|
$("meta[property]").each((_, el) => {
|
|
@@ -93,9 +62,7 @@ async function processArticleFile(filePath) {
|
|
|
93
62
|
else
|
|
94
63
|
console.warn(`No OG description found for ${item.url}`);
|
|
95
64
|
// Add a small delay to avoid overwhelming servers
|
|
96
|
-
await
|
|
97
|
-
setTimeout(resolve, 1000);
|
|
98
|
-
});
|
|
65
|
+
await setTimeout(1000);
|
|
99
66
|
}
|
|
100
67
|
// Write back to file
|
|
101
68
|
writeFileSync(filePath, `${JSON.stringify(articleData, null, "\t")}\n`, "utf-8");
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"update-json-articles.js","sourceRoot":"","sources":["../src/update-json-articles.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AACrE,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,
|
|
1
|
+
{"version":3,"file":"update-json-articles.js","sourceRoot":"","sources":["../src/update-json-articles.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AACrE,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACnE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAkBnD,KAAK,UAAU,SAAS,CACvB,GAAW;IAEX,IAAI,CAAC;QACJ,OAAO,CAAC,GAAG,CAAC,yBAAyB,GAAG,EAAE,CAAC,CAAC;QAE5C,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YACjC,OAAO,EAAE;gBACR,YAAY,EACX,2HAA2H;aAC5H;SACD,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CAAC,uBAAuB,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QAC3D,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;QACzD,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAElD,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,MAAM,MAAM,GAA2B,EAAE,CAAC;QAC1C,CAAC,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;YAClC,MAAM,QAAQ,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,CAAC;YAChD,MAAM,OAAO,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACtC,IAAI,QAAQ,IAAI,OAAO,EAAE,CAAC;gBACzB,MAAM,CAAC,QAAQ,CAAC,GAAG,OAAO,CAAC;YAC5B,CAAC;QACF,CAAC,CAAC,CAAC;QAEH,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;QACtC,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;QACnC,MAAM,aAAa,GAAG,MAAM,CAAC,gBAAgB,CAAC,CAAC;QAE/C,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC;IAC/C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,OAAO,CAAC,KAAK,CAAC,8BAA8B,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;QAC3D,OAAO,EAAE,CAAC;IACX,CAAC;AACF,CAAC;AAED,KAAK,UAAU,kBAAkB,CAAC,QAAgB;IACjD,IAAI,CAAC;QACJ,MAAM,WAAW,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACpD,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAiB,CAAC;QAE5D,OAAO,CAAC,GAAG,CAAC,cAAc,QAAQ,KAAK,CAAC,CAAC;QAEzC,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,IAAI,EAAE,CAAC;YACrC,gCAAgC;YAChC,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;gBACxE,SAAS;YACV,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAEzC,0CAA0C;YAC1C,IAAI,MAAM,CAAC,UAAU;gBAAE,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;;gBACtD,OAAO,CAAC,IAAI,CAAC,yBAAyB,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;YAEvD,IAAI,MAAM,CAAC,OAAO;gBAAE,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;;gBAC7C,OAAO,CAAC,IAAI,CAAC,yBAAyB,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;YAEvD,IAAI,MAAM,CAAC,aAAa;gBAAE,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC;;gBAC/D,OAAO,CAAC,IAAI,CAAC,+BAA+B,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;YAE7D,kDAAkD;YAClD,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC;QACxB,CAAC;QAED,qBAAqB;QACrB,aAAa,CACZ,QAAQ,EACR,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAC9C,OAAO,CACP,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,wBAAwB,QAAQ,EAAE,CAAC,CAAC;IACjD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,OAAO,CAAC,KAAK,CAAC,oBAAoB,QAAQ,GAAG,EAAE,KAAK,CAAC,CAAC;IACvD,CAAC;AACF,CAAC;AAED,KAAK,UAAU,IAAI;IAClB,MAAM,GAAG,GAAG;QACX,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY;QACtC,iBAAiB,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB;QAChD,kBAAkB,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB;KACzC,CAAC;IAEX,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IAChE,CAAC;IAED,MAAM,mBAAmB,GAAG,qBAAqB,CAAC;QACjD,GAAG,EAAE,GAAG,CAAC,YAAY,IAAI,EAAE;QAC3B,OAAO,EAAE,GAAG,CAAC,iBAAiB,IAAI,EAAE;QACpC,QAAQ,EAAE,GAAG,CAAC,kBAAkB,IAAI,EAAE;KACtC,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;IAE1D,IAAI,CAAC;QACJ,MAAM,KAAK,GAAG,WAAW,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CACrD,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CACtB,CAAC;QAEF,OAAO,CAAC,GAAG,CAAC,SAAS,KAAK,CAAC,MAAM,wBAAwB,CAAC,CAAC;QAE3D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YAC1B,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;YACxC,MAAM,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QACpC,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAC;QACjD,MAAM,mBAAmB,CAAC,UAAU,CAAC,gCAAgC,EAAE;YACtE,MAAM,EAAE,sBAAsB;SAC9B,CAAC,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,OAAO,CAAC,KAAK,CAAC,wBAAwB,EAAE,KAAK,CAAC,CAAC;QAC/C,MAAM,mBAAmB,CAAC,WAAW,CACpC,gCAAgC,MAAM,CAAC,KAAK,CAAC,EAAE,EAC/C;YACC,MAAM,EAAE,sBAAsB;SAC9B,CACD,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;AACF,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;IAC/B,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC,CAAC,CAAC"}
|
|
@@ -1,43 +1,12 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { createPushoverService } from "@s-hirano-ist/s-notification";
|
|
3
|
-
import iconv from "iconv-lite";
|
|
4
3
|
import { existsSync } from "node:fs";
|
|
5
4
|
import { mkdir, readdir, readFile, writeFile } from "node:fs/promises";
|
|
6
5
|
import path from "node:path";
|
|
7
6
|
import TurndownService from "turndown";
|
|
7
|
+
import { decodeHtml } from "./lib/html-charset.js";
|
|
8
8
|
const JSON_DIR = "json/article";
|
|
9
9
|
const OUTPUT_DIR = "raw/article";
|
|
10
|
-
function normalizeCharset(charset) {
|
|
11
|
-
const normalized = charset.toLowerCase().replaceAll(/[^a-z0-9]/gu, "");
|
|
12
|
-
const mapping = {
|
|
13
|
-
shiftjis: "Shift_JIS",
|
|
14
|
-
sjis: "Shift_JIS",
|
|
15
|
-
xsjis: "Shift_JIS",
|
|
16
|
-
eucjp: "EUC-JP",
|
|
17
|
-
xeucjp: "EUC-JP",
|
|
18
|
-
};
|
|
19
|
-
return mapping[normalized] || charset;
|
|
20
|
-
}
|
|
21
|
-
function detectCharset(headers, buffer) {
|
|
22
|
-
// 1. Content-Type ヘッダーから検出
|
|
23
|
-
const contentType = headers.get("content-type") || "";
|
|
24
|
-
const headerMatch = /charset=([^\s;]+)/iu.exec(contentType);
|
|
25
|
-
if (headerMatch)
|
|
26
|
-
return normalizeCharset(headerMatch[1]);
|
|
27
|
-
// 2. HTML meta タグから検出(ASCII範囲で仮デコード)
|
|
28
|
-
const preview = buffer
|
|
29
|
-
.subarray(0, Math.min(4096, buffer.length))
|
|
30
|
-
.toString("ascii");
|
|
31
|
-
// <meta charset="...">
|
|
32
|
-
const metaCharset = /<meta\s{1,200}charset=["']?([^"'\s>]+)/iu.exec(preview);
|
|
33
|
-
if (metaCharset)
|
|
34
|
-
return normalizeCharset(metaCharset[1]);
|
|
35
|
-
// <meta http-equiv="Content-Type" content="...; charset=...">
|
|
36
|
-
const metaHttpEquiv = /<meta[^>]{1,500}http-equiv=["']?Content-Type["']?[^>]{1,500}content=["'][^"']{0,500}charset=([^"'\s;]+)/iu.exec(preview);
|
|
37
|
-
if (metaHttpEquiv)
|
|
38
|
-
return normalizeCharset(metaHttpEquiv[1]);
|
|
39
|
-
return "utf-8";
|
|
40
|
-
}
|
|
41
10
|
async function fetchWebsiteMarkdown(url) {
|
|
42
11
|
try {
|
|
43
12
|
const response = await fetch(url, {
|
|
@@ -49,8 +18,7 @@ async function fetchWebsiteMarkdown(url) {
|
|
|
49
18
|
throw new Error(`HTTP error! status: ${response.status}`);
|
|
50
19
|
}
|
|
51
20
|
const buffer = Buffer.from(await response.arrayBuffer());
|
|
52
|
-
const
|
|
53
|
-
const html = iconv.decode(buffer, charset);
|
|
21
|
+
const html = decodeHtml(response.headers, buffer);
|
|
54
22
|
const turndownService = new TurndownService({
|
|
55
23
|
headingStyle: "atx",
|
|
56
24
|
codeBlockStyle: "fenced",
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"update-raw-articles.js","sourceRoot":"","sources":["../src/update-raw-articles.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AACrE,OAAO,
|
|
1
|
+
{"version":3,"file":"update-raw-articles.js","sourceRoot":"","sources":["../src/update-raw-articles.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AACrE,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACvE,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,eAAe,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAEnD,MAAM,QAAQ,GAAG,cAAc,CAAC;AAChC,MAAM,UAAU,GAAG,aAAa,CAAC;AAEjC,KAAK,UAAU,oBAAoB,CAAC,GAAW;IAC9C,IAAI,CAAC;QACJ,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YACjC,OAAO,EAAE;gBACR,YAAY,EACX,2HAA2H;aAC5H;SACD,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CAAC,uBAAuB,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QAC3D,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;QACzD,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAElD,MAAM,eAAe,GAAG,IAAI,eAAe,CAAC;YAC3C,YAAY,EAAE,KAAK;YACnB,cAAc,EAAE,QAAQ;YACxB,gBAAgB,EAAE,GAAG;YACrB,eAAe,EAAE,IAAI;YACrB,WAAW,EAAE,GAAG;SAChB,CAAC,CAAC;QAEH,eAAe,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;QAEtE,eAAe,CAAC,OAAO,CAAC,eAAe,EAAE;YACxC,MAAM,EAAE,CAAC,GAAG,CAAC;YACb,WAAW,EAAE,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE;gBAC9B,MAAM,OAAO,GAAG,IAAyB,CAAC;gBAC1C,MAAM,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;gBAC1C,IAAI,CAAC,IAAI;oBAAE,OAAO,OAAO,CAAC;gBAE1B,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;oBACzD,OAAO,IAAI,OAAO,KAAK,IAAI,GAAG,CAAC;gBAChC,CAAC;gBACD,IAAI,CAAC;oBACJ,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC;oBAC5C,OAAO,IAAI,OAAO,KAAK,WAAW,GAAG,CAAC;gBACvC,CAAC;gBAAC,MAAM,CAAC;oBACR,OAAO,OAAO,CAAC;gBAChB,CAAC;YACF,CAAC;SACD,CAAC,CAAC;QAEH,eAAe,CAAC,OAAO,CAAC,gBAAgB,EAAE;YACzC,MAAM,EAAE,CAAC,KAAK,CAAC;YACf,WAAW,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE;gBAC/B,MAAM,OAAO,GAAG,IAAwB,CAAC;gBACzC,MAAM,GAAG,GAAG,OAAO,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;gBACxC,MAAM,GAAG,GAAG,OAAO,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;gBAC9C,IAAI,CAAC,GAAG;oBAAE,OAAO,EAAE,CAAC;gBAEpB,IAAI,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;oBACvD,OAAO,KAAK,GAAG,KAAK,GAAG,GAAG,CAAC;gBAC5B,CAAC;gBACD,IAAI,CAAC;oBACJ,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC;oBAC3C,OAAO,KAAK,GAAG,KAAK,WAAW,GAAG,CAAC;gBACpC,CAAC;gBAAC,MAAM,CAAC;oBACR,OAAO,EAAE,CAAC;gBACX,CAAC;YACF,CAAC;SACD,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAChD,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;IACxB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,OAAO,CAAC,KAAK,CAAC,mBAAmB,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;QAChD,OAAO,EAAE,CAAC;IACX,CAAC;AACF,CAAC;AAeD,KAAK,UAAU,cAAc,CAAC,QAAgB;IAC7C,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAClD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAiB,CAAC;IAEjD,sJAAsJ;IACtJ,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;IAEjC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC7B,MAAM,KAAK,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9C,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC7B,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC;QACzC,IAAI,CAAC;YACJ,MAAM,WAAW,GAAG,GAAG,CAAC,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC;YACrD,MAAM,OAAO,GAAG,kBAAkB,CAAC,WAAW,CAAC,CAAC;YAChD,MAAM,cAAc,GAAG,GAAG,OAAO,KAAK,CAAC;YACvC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;YAEzD,IAAI,UAAU,CAAC,UAAU,CAAC,IAAI,IAAI,EAAE,CAAC;gBACpC,SAAS;YACV,CAAC;YAED,MAAM,WAAW,GAAG,MAAM,oBAAoB,CAAC,GAAG,CAAC,CAAC;YAEpD,IAAI,CAAC,WAAW,EAAE,CAAC;gBAClB,SAAS;YACV,CAAC;YAED,MAAM,eAAe,GAAG,MAAM,KAAK,KAAK,GAAG;;;;;;EAM5C,KAAK;;;;EAIL,WAAW;CACZ,CAAC;YAEC,MAAM,SAAS,CAAC,UAAU,EAAE,eAAe,EAAE,OAAO,CAAC,CAAC;YACtD,OAAO,CAAC,GAAG,CAAC,aAAa,UAAU,EAAE,CAAC,CAAC;QACxC,CAAC;QAAC,MAAM,CAAC;YACR,OAAO,CAAC,KAAK,CAAC,gBAAgB,EAAE,GAAG,CAAC,CAAC;QACtC,CAAC;IACF,CAAC;AACF,CAAC;AAED,KAAK,UAAU,IAAI;IAClB,MAAM,GAAG,GAAG;QACX,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY;QACtC,iBAAiB,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB;QAChD,kBAAkB,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB;KACzC,CAAC;IAEX,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IAChE,CAAC;IAED,MAAM,mBAAmB,GAAG,qBAAqB,CAAC;QACjD,GAAG,EAAE,GAAG,CAAC,YAAY,IAAI,EAAE;QAC3B,OAAO,EAAE,GAAG,CAAC,iBAAiB,IAAI,EAAE;QACpC,QAAQ,EAAE,GAAG,CAAC,kBAAkB,IAAI,EAAE;KACtC,CAAC,CAAC;IAEH,IAAI,CAAC;QACJ,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC1C,MAAM,aAAa,GAAG,SAAS;aAC7B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;aACxC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC;QAE3C,KAAK,MAAM,QAAQ,IAAI,aAAa,EAAE,CAAC;YACtC,OAAO,CAAC,GAAG,CAAC,eAAe,QAAQ,EAAE,CAAC,CAAC;YACvC,MAAM,cAAc,CAAC,QAAQ,CAAC,CAAC;QAChC,CAAC;QACD,MAAM,mBAAmB,CAAC,UAAU,CAAC,+BAA+B,EAAE;YACrE,MAAM,EAAE,qBAAqB;SAC7B,CAAC,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,OAAO,CAAC,KAAK,CAAC,wBAAwB,EAAE,KAAK,CAAC,CAAC;QAC/C,MAAM,mBAAmB,CAAC,WAAW,CACpC,+BAA+B,MAAM,CAAC,KAAK,CAAC,EAAE,EAC9C;YACC,MAAM,EAAE,qBAAqB;SAC7B,CACD,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;AACF,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;IAC/B,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@s-hirano-ist/s-scripts",
|
|
3
|
-
"version": "1.26.
|
|
3
|
+
"version": "1.26.2",
|
|
4
4
|
"description": "CLI scripts for s-private data operations",
|
|
5
|
+
"engines": {
|
|
6
|
+
"node": ">=24"
|
|
7
|
+
},
|
|
5
8
|
"type": "module",
|
|
6
9
|
"bin": {
|
|
7
10
|
"s-content-format": "./dist/s-content-format.js",
|
|
@@ -51,16 +54,14 @@
|
|
|
51
54
|
"dependencies": {
|
|
52
55
|
"@googleapis/books": "9.0.0",
|
|
53
56
|
"cheerio": "1.2.0",
|
|
54
|
-
"glob": "13.0.6",
|
|
55
|
-
"iconv-lite": "0.7.3",
|
|
56
57
|
"js-yaml": "5.3.0",
|
|
57
|
-
"oxfmt": "0.
|
|
58
|
+
"oxfmt": "0.65.0",
|
|
58
59
|
"turndown": "7.2.4",
|
|
59
|
-
"@s-hirano-ist/s-core": "1.21.
|
|
60
|
-
"@s-hirano-ist/s-database": "1.21.
|
|
60
|
+
"@s-hirano-ist/s-core": "1.21.3",
|
|
61
|
+
"@s-hirano-ist/s-database": "1.21.1",
|
|
62
|
+
"@s-hirano-ist/s-search": "1.18.13",
|
|
61
63
|
"@s-hirano-ist/s-notification": "1.18.5",
|
|
62
64
|
"@s-hirano-ist/s-image-processing": "1.0.0",
|
|
63
|
-
"@s-hirano-ist/s-search": "1.18.13",
|
|
64
65
|
"@s-hirano-ist/s-storage": "1.18.9"
|
|
65
66
|
},
|
|
66
67
|
"devDependencies": {
|