@mz1999/defuddle 0.14.1

package/dist/utils/transcript.js

@@ -0,0 +1,61 @@
+"use strict";
+/**
+ * Standardized transcript HTML and text construction.
+ *
+ * Used by YouTube (and potentially other video/audio extractors)
+ * to produce consistent transcript markup.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.formatTimestamp = formatTimestamp;
+exports.buildTranscript = buildTranscript;
+const dom_1 = require("./dom");
+/**
+ * Format seconds as a human-readable timestamp (M:SS or H:MM:SS).
+ */
+function formatTimestamp(seconds) {
+    const h = Math.floor(seconds / 3600);
+    const m = Math.floor((seconds % 3600) / 60);
+    const s = Math.floor(seconds % 60);
+    if (h > 0) {
+        return `${h}:${String(m).padStart(2, '0')}:${String(s).padStart(2, '0')}`;
+    }
+    return `${m}:${String(s).padStart(2, '0')}`;
+}
+/**
+ * Build transcript HTML and text from segments and optional chapters.
+ *
+ * @param site - Site identifier for wrapper class (e.g. "youtube")
+ * @param segments - Grouped transcript segments with timestamps and speaker info
+ * @param chapters - Optional chapter headings with start times
+ */
+function buildTranscript(site, segments, chapters = []) {
+    const sortedChapters = [...chapters].sort((a, b) => a.start - b.start);
+    let chapterIdx = 0;
+    const htmlParts = [];
+    const textParts = [];
+    for (const segment of segments) {
+        // Insert chapter headings before this segment
+        while (chapterIdx < sortedChapters.length && sortedChapters[chapterIdx].start <= segment.start) {
+            const title = sortedChapters[chapterIdx].title;
+            htmlParts.push(`<h3>${(0, dom_1.escapeHtml)(title)}</h3>`);
+            if (textParts.length > 0)
+                textParts.push('');
+            textParts.push(`### ${title}`);
+            textParts.push('');
+            chapterIdx++;
+        }
+        const timestamp = formatTimestamp(segment.start);
+        const speakerClass = segment.speaker !== undefined ? ` speaker-${segment.speaker}` : '';
+        const tsHtml = `<strong><span class="timestamp" data-timestamp="${segment.start}">${timestamp}</span></strong>`;
+        htmlParts.push(`<p class="transcript-segment${speakerClass}">${tsHtml} · ${(0, dom_1.escapeHtml)(segment.text)}</p>`);
+        if (segment.speakerChange && textParts.length > 0) {
+            textParts.push('');
+        }
+        textParts.push(`**${timestamp}** · ${segment.text}`);
+    }
+    return {
+        html: `<div class="${site} transcript">\n<h2>Transcript</h2>\n${htmlParts.join('\n')}\n</div>`,
+        text: textParts.join('\n'),
+    };
+}
+//# sourceMappingURL=transcript.js.map

package/dist/utils/transcript.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"transcript.js","sourceRoot":"","sources":["../../src/utils/transcript.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;AA6BH,0CASC;AASD,0CAqCC;AAlFD,+BAAmC;AAwBnC;;GAEG;AACH,SAAgB,eAAe,CAAC,OAAe;IAC9C,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC;IACrC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IAC5C,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;IAEnC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACX,OAAO,GAAG,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;IAC3E,CAAC;IACD,OAAO,GAAG,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;AAC7C,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,eAAe,CAC9B,IAAY,EACZ,QAA6B,EAC7B,WAAgC,EAAE;IAElC,MAAM,cAAc,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IACvE,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,MAAM,SAAS,GAAa,EAAE,CAAC;IAE/B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAChC,8CAA8C;QAC9C,OAAO,UAAU,GAAG,cAAc,CAAC,MAAM,IAAI,cAAc,CAAC,UAAU,CAAC,CAAC,KAAK,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YAChG,MAAM,KAAK,GAAG,cAAc,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC;YAC/C,SAAS,CAAC,IAAI,CAAC,OAAO,IAAA,gBAAU,EAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAChD,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC;gBAAE,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC7C,SAAS,CAAC,IAAI,CAAC,OAAO,KAAK,EAAE,CAAC,CAAC;YAC/B,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACnB,UAAU,EAAE,CAAC;QACd,CAAC;QAED,MAAM,SAAS,GAAG,eAAe,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACjD,MAAM,YAAY,GAAG,OAAO,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,YAAY,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxF,MAAM,MAAM,GAAG,mDAAmD,OAAO,CAAC,KAAK,KAAK,SAAS,kBAAkB,CAAC;QAChH,SAAS,CAAC,IAAI,CAAC,+BAA+B,YAAY,KAAK,MAAM,MAAM,IAAA,gBAAU,EAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAE3G,IAAI,OAAO,CAAC,aAAa,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnD,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACpB,CAAC;QACD,SAAS,CAAC,IAAI,CAAC,KAAK,SAAS,QAAQ,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IACtD,CAAC;IAED,OAAO;QACN,IAAI,EAAE,eAAe,IAAI,uCAAuC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU;QAC9F,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;KAC1B,CAAC;AACH,CAAC"}

package/dist/utils.d.ts

@@ -0,0 +1,13 @@
+export declare function isElement(node: Node): node is Element;
+export declare function isTextNode(node: Node): node is Text;
+export declare function isCommentNode(node: Node): node is Comment;
+export declare function getComputedStyle(element: Element): CSSStyleDeclaration | null;
+export declare function getWindow(doc: Document): Window | null;
+export declare function textPreview(el: Element): string;
+export declare function logDebug(debug: boolean, message: string, ...args: any[]): void;
+/**
+ * Count words in text, handling CJK characters (Chinese, Japanese, Korean).
+ * CJK characters are counted individually since they don't use spaces between words.
+ * Non-CJK text is counted by splitting on whitespace.
+ */
+export declare function countWords(text: string): number;

package/dist/utils.js

@@ -0,0 +1,98 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isElement = isElement;
+exports.isTextNode = isTextNode;
+exports.isCommentNode = isCommentNode;
+exports.getComputedStyle = getComputedStyle;
+exports.getWindow = getWindow;
+exports.textPreview = textPreview;
+exports.logDebug = logDebug;
+exports.countWords = countWords;
+const NODE_TYPE = {
+    ELEMENT_NODE: 1,
+    ATTRIBUTE_NODE: 2,
+    TEXT_NODE: 3,
+    CDATA_SECTION_NODE: 4,
+    ENTITY_REFERENCE_NODE: 5,
+    ENTITY_NODE: 6,
+    PROCESSING_INSTRUCTION_NODE: 7,
+    COMMENT_NODE: 8,
+    DOCUMENT_NODE: 9,
+    DOCUMENT_TYPE_NODE: 10,
+    DOCUMENT_FRAGMENT_NODE: 11,
+    NOTATION_NODE: 12
+};
+function isElement(node) {
+    return node.nodeType === NODE_TYPE.ELEMENT_NODE;
+}
+function isTextNode(node) {
+    return node.nodeType === NODE_TYPE.TEXT_NODE;
+}
+function isCommentNode(node) {
+    return node.nodeType === NODE_TYPE.COMMENT_NODE;
+}
+function getComputedStyle(element) {
+    const win = getWindow(element.ownerDocument);
+    if (!win)
+        return null;
+    return win.getComputedStyle(element);
+}
+function getWindow(doc) {
+    // First try defaultView
+    if (doc.defaultView) {
+        return doc.defaultView;
+    }
+    // Then try ownerWindow
+    if (doc.ownerWindow) {
+        return doc.ownerWindow;
+    }
+    // Finally try to get window from document
+    if (doc.window) {
+        return doc.window;
+    }
+    return null;
+}
+function textPreview(el) {
+    return (el.textContent || '').trim().substring(0, 200);
+}
+function logDebug(debug, message, ...args) {
+    if (debug) {
+        console.log('Defuddle:', message, ...args);
+    }
+}
+/**
+ * Count words in text, handling CJK characters (Chinese, Japanese, Korean).
+ * CJK characters are counted individually since they don't use spaces between words.
+ * Non-CJK text is counted by splitting on whitespace.
+ */
+function countWords(text) {
+    if (!text)
+        return 0;
+    let cjkCount = 0;
+    let wordCount = 0;
+    let inWord = false;
+    for (let i = 0; i < text.length; i++) {
+        const code = text.charCodeAt(i);
+        // Check for CJK character ranges (BMP only — Extension B+ are
+        // surrogate pairs and would need codePointAt, rare in practice)
+        if ((code >= 0x3040 && code <= 0x309f) || // Hiragana
+            (code >= 0x30a0 && code <= 0x30ff) || // Katakana
+            (code >= 0x3400 && code <= 0x4dbf) || // CJK Extension A
+            (code >= 0x4e00 && code <= 0x9fff) || // CJK Unified Ideographs
+            (code >= 0xf900 && code <= 0xfaff) || // CJK Compatibility Ideographs
+            (code >= 0xac00 && code <= 0xd7af) // Korean Hangul
+        ) {
+            cjkCount++;
+            inWord = false;
+        }
+        else if (code <= 32) {
+            inWord = false;
+        }
+        else if (!inWord) {
+            wordCount++;
+            inWord = true;
+        }
+    }
+    return cjkCount + wordCount;
+}
+//# sourceMappingURL=utils.js.map

package/dist/utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":";;AAeA,8BAEC;AAED,gCAEC;AAED,sCAEC;AAED,4CAIC;AAED,8BAiBC;AAED,kCAEC;AAED,4BAIC;AAOD,gCA+BC;AAlGD,MAAM,SAAS,GAAG;IACjB,YAAY,EAAE,CAAC;IACf,cAAc,EAAE,CAAC;IACjB,SAAS,EAAE,CAAC;IACZ,kBAAkB,EAAE,CAAC;IACrB,qBAAqB,EAAE,CAAC;IACxB,WAAW,EAAE,CAAC;IACd,2BAA2B,EAAE,CAAC;IAC9B,YAAY,EAAE,CAAC;IACf,aAAa,EAAE,CAAC;IAChB,kBAAkB,EAAE,EAAE;IACtB,sBAAsB,EAAE,EAAE;IAC1B,aAAa,EAAE,EAAE;CACjB,CAAC;AAEF,SAAgB,SAAS,CAAC,IAAU;IACnC,OAAO,IAAI,CAAC,QAAQ,KAAK,SAAS,CAAC,YAAY,CAAC;AACjD,CAAC;AAED,SAAgB,UAAU,CAAC,IAAU;IACpC,OAAO,IAAI,CAAC,QAAQ,KAAK,SAAS,CAAC,SAAS,CAAC;AAC9C,CAAC;AAED,SAAgB,aAAa,CAAC,IAAU;IACvC,OAAO,IAAI,CAAC,QAAQ,KAAK,SAAS,CAAC,YAAY,CAAC;AACjD,CAAC;AAED,SAAgB,gBAAgB,CAAC,OAAgB;IAChD,MAAM,GAAG,GAAG,SAAS,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;IAC7C,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IACtB,OAAO,GAAG,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;AACtC,CAAC;AAED,SAAgB,SAAS,CAAC,GAAa;IACtC,wBAAwB;IACxB,IAAI,GAAG,CAAC,WAAW,EAAE,CAAC;QACrB,OAAO,GAAG,CAAC,WAAW,CAAC;IACxB,CAAC;IAED,uBAAuB;IACvB,IAAK,GAAW,CAAC,WAAW,EAAE,CAAC;QAC9B,OAAQ,GAAW,CAAC,WAAW,CAAC;IACjC,CAAC;IAED,0CAA0C;IAC1C,IAAK,GAAW,CAAC,MAAM,EAAE,CAAC;QACzB,OAAQ,GAAW,CAAC,MAAM,CAAC;IAC5B,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC;AAED,SAAgB,WAAW,CAAC,EAAW;IACtC,OAAO,CAAC,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AACxD,CAAC;AAED,SAAgB,QAAQ,CAAC,KAAc,EAAE,OAAe,EAAE,GAAG,IAAW;IACvE,IAAI,KAAK,EAAE,CAAC;QACX,OAAO,CAAC,GAAG,CAAC,WAAW,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,CAAC;IAC5C,CAAC;AACF,CAAC;AAED;;;;GAIG;AACH,SAAgB,UAAU,CAAC,IAAY;IACtC,IAAI,CAAC,IAAI;QAAE,OAAO,CAAC,CAAC;IAEpB,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,MAAM,GAAG,KAAK,CAAC;IAEnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAEhC,8DAA8D;QAC9D,gEAAgE;QAChE,IACC,CAAC,IAAI,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,CAAC,IAAI,WAAW;YACjD,CAAC,IAAI,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,CAAC,IAAI,WAAW;YACjD,CAAC,IAAI,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,CAAC,IAAI,kBAAkB;YACxD,CAAC,IAAI,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,CAAC,IAAI,yBAAyB;YAC/D,CAAC,IAAI,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,CAAC,IAAI,+BAA+B;YACrE,CAAC,IAAI,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,CAAC,CAAI,gBAAgB;UACrD,CAAC;YACF,QAAQ,EAAE,CAAC;YACX,MAAM,GAAG,KAAK,CAAC;QAChB,CAAC;aAAM,IAAI,IAAI,IAAI,EAAE,EAAE,CAAC;YACvB,MAAM,GAAG,KAAK,CAAC;QAChB,CAAC;aAAM,IAAI,CAAC,MAAM,EAAE,CAAC;YACpB,SAAS,EAAE,CAAC;YACZ,MAAM,GAAG,IAAI,CAAC;QACf,CAAC;IACF,CAAC;IAED,OAAO,QAAQ,GAAG,SAAS,CAAC;AAC7B,CAAC"}

package/package.json

@@ -0,0 +1,107 @@
+{
+	"name": "@mz1999/defuddle",
+	"version": "0.14.1",
+	"description": "Extract article content and metadata from web pages (with HTTP proxy support).",
+	"publishConfig": {
+		"access": "public"
+	},
+	"main": "dist/index.js",
+	"bin": {
+		"defuddle": "dist/cli.js"
+	},
+	"module": "dist/index.js",
+	"types": "dist/index.d.ts",
+	"typesVersions": {
+		"*": {
+			"*": [
+				"dist/index.d.ts"
+			],
+			"full": [
+				"dist/index.full.d.ts"
+			],
+			"node": [
+				"dist/node.d.ts"
+			]
+		}
+	},
+	"exports": {
+		".": {
+			"types": "./dist/index.d.ts",
+			"import": "./dist/index.js",
+			"require": "./dist/index.js"
+		},
+		"./full": {
+			"types": "./dist/index.full.d.ts",
+			"import": "./dist/index.full.js",
+			"require": "./dist/index.full.js"
+		},
+		"./node": {
+			"types": "./dist/node.d.ts",
+			"import": "./dist/node.js"
+		}
+	},
+	"scripts": {
+		"clean": "rm -rf dist",
+		"build:types": "tsc --project tsconfig.declarations.json",
+		"build:js": "webpack",
+		"build:node": "tsc --project tsconfig.node.json",
+		"build": "npm run clean && npm run build:types && npm run build:node && npm run build:js",
+		"prepublishOnly": "npm run build",
+		"dev:types": "tsc --project tsconfig.declarations.json --watch",
+		"dev:js": "webpack --watch --mode development",
+		"dev:node": "tsc --project tsconfig.node.json --watch",
+		"dev": "npm run clean && npm run build:types && concurrently \"npm run dev:types\" \"npm run dev:node\" \"npm run dev:js\"",
+		"test": "TZ=UTC vitest run",
+		"test:jsdom": "TZ=UTC DOM=jsdom vitest run",
+		"playground": "node playground/server.js"
+	},
+	"keywords": [
+		"readability",
+		"content-extraction",
+		"article-extraction",
+		"web-scraping",
+		"html-cleanup",
+		"content-parser",
+		"article-parser",
+		"dom"
+	],
+	"author": "kepano (original), mazhen (proxy fork)",
+	"license": "MIT",
+	"repository": {
+		"type": "git",
+		"url": "git+https://github.com/mazhen/defuddle.git"
+	},
+	"bugs": {
+		"url": "https://github.com/mazhen/defuddle/issues"
+	},
+	"homepage": "https://github.com/mazhen/defuddle",
+	"dependencies": {
+		"commander": "^12.1.0",
+		"undici": "^6.0.0"
+	},
+	"optionalDependencies": {
+		"linkedom": "^0.18.12",
+		"mathml-to-latex": "^1.5.0",
+		"temml": "^0.13.1",
+		"turndown": "^7.2.0"
+	},
+	"devDependencies": {
+		"@types/jsdom": "^21.1.6",
+		"@types/node": "^20.19.0",
+		"@types/turndown": "^5.0.5",
+		"concurrently": "^8.2.2",
+		"jsdom": "^24.0.0",
+		"terser-webpack-plugin": "^5.3.14",
+		"ts-loader": "^9.5.1",
+		"typescript": "^5.3.3",
+		"undici-types": "^6.0.0",
+		"vitest": "^3.2.4",
+		"webpack": "^5.90.3",
+		"webpack-cli": "^5.1.4"
+	},
+	"files": [
+		"dist",
+		"README.md",
+		"LICENSE"
+	]
+}