defuddle 0.8.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -49
- package/dist/constants.js +37 -15
- package/dist/constants.js.map +1 -1
- package/dist/defuddle.d.ts +34 -0
- package/dist/defuddle.js +371 -29
- package/dist/defuddle.js.map +1 -1
- package/dist/elements/code.js +28 -4
- package/dist/elements/code.js.map +1 -1
- package/dist/elements/footnotes.js +315 -17
- package/dist/elements/footnotes.js.map +1 -1
- package/dist/elements/images.js +6 -6
- package/dist/elements/images.js.map +1 -1
- package/dist/elements/math.base.js +11 -5
- package/dist/elements/math.base.js.map +1 -1
- package/dist/elements/math.core.js +9 -16
- package/dist/elements/math.core.js.map +1 -1
- package/dist/elements/math.js +0 -2
- package/dist/elements/math.js.map +1 -1
- package/dist/extractor-registry.d.ts +2 -0
- package/dist/extractor-registry.js +16 -4
- package/dist/extractor-registry.js.map +1 -1
- package/dist/extractors/_base.d.ts +2 -0
- package/dist/extractors/_base.js +6 -0
- package/dist/extractors/_base.js.map +1 -1
- package/dist/extractors/_conversation.js +2 -1
- package/dist/extractors/_conversation.js.map +1 -1
- package/dist/extractors/chatgpt.js +4 -3
- package/dist/extractors/chatgpt.js.map +1 -1
- package/dist/extractors/claude.js +3 -2
- package/dist/extractors/claude.js.map +1 -1
- package/dist/extractors/gemini.js +5 -4
- package/dist/extractors/gemini.js.map +1 -1
- package/dist/extractors/github.js +7 -6
- package/dist/extractors/github.js.map +1 -1
- package/dist/extractors/grok.js +2 -1
- package/dist/extractors/grok.js.map +1 -1
- package/dist/extractors/hackernews.js +5 -3
- package/dist/extractors/hackernews.js.map +1 -1
- package/dist/extractors/reddit.d.ts +7 -0
- package/dist/extractors/reddit.js +108 -4
- package/dist/extractors/reddit.js.map +1 -1
- package/dist/extractors/twitter.js +5 -3
- package/dist/extractors/twitter.js.map +1 -1
- package/dist/extractors/x-article.js +2 -1
- package/dist/extractors/x-article.js.map +1 -1
- package/dist/extractors/x-oembed.d.ts +21 -0
- package/dist/extractors/x-oembed.js +357 -0
- package/dist/extractors/x-oembed.js.map +1 -0
- package/dist/index.full.d.ts +3 -0
- package/dist/index.full.js +1 -1
- package/dist/index.js +1 -1
- package/dist/markdown.d.ts +2 -4
- package/dist/markdown.js +76 -20
- package/dist/markdown.js.map +1 -1
- package/dist/metadata.d.ts +1 -0
- package/dist/metadata.js +46 -21
- package/dist/metadata.js.map +1 -1
- package/dist/node.js +1 -1
- package/dist/node.js.map +1 -1
- package/dist/scoring.js +94 -22
- package/dist/scoring.js.map +1 -1
- package/dist/standardize.js +206 -38
- package/dist/standardize.js.map +1 -1
- package/dist/types.d.ts +6 -0
- package/dist/utils/dom.d.ts +23 -0
- package/dist/utils/dom.js +62 -0
- package/dist/utils/dom.js.map +1 -0
- package/package.json +9 -9
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"github.js","sourceRoot":"","sources":["../../src/extractors/github.ts"],"names":[],"mappings":";;;AAAA,mCAAwC;
|
|
1
|
+
{"version":3,"file":"github.js","sourceRoot":"","sources":["../../src/extractors/github.ts"],"names":[],"mappings":";;;AAAA,mCAAwC;AAExC,sCAAwD;AAExD,MAAa,eAAgB,SAAQ,qBAAa;IACjD,UAAU;QACT,MAAM,gBAAgB,GAAG;YACxB,sDAAsD;YACtD,6BAA6B;YAC7B,wCAAwC;YACxC,oBAAoB;YACpB,yBAAyB;SACzB,CAAC;QAEF,MAAM,oBAAoB,GAAG;YAC5B,KAAK,EAAE;gBACN,uCAAuC;gBACvC,6BAA6B;aAC7B;SACD,CAAA;QAED,OAAO,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC;eACpF,MAAM,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;IACvI,CAAC;IAED,OAAO;QACN,OAAO,IAAI,CAAC,YAAY,EAAE,CAAC;IAC5B,CAAC;IAEO,YAAY;QACnB,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,EAAE,CAAC;QACxC,MAAM,WAAW,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAE9C,IAAI,OAAO,GAAG,EAAE,CAAC;QAEjB,oCAAoC;QACpC,MAAM,cAAc,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,8CAA8C,CAAC,CAAC;QACnG,IAAI,cAAc,EAAE,CAAC;YACpB,MAAM,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,cAAc,EAAE;gBACtD,2CAA2C;gBAC3C,uDAAuD;gBACvD,2CAA2C;gBAC3C,mDAAmD;gBACnD,0BAA0B;aAC1B,CAAC,CAAC;YAEH,MAAM,gBAAgB,GAAG,cAAc,CAAC,aAAa,CAAC,eAAe,CAAC,CAAC;YACvE,MAAM,cAAc,GAAG,gBAAgB,EAAE,YAAY,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;YAExE,MAAM,gBAAgB,GAAG,cAAc,CAAC,aAAa,CAAC,kDAAkD,CAAC,CAAC;YAE1G,IAAI,gBAAgB,EAAE,CAAC;gBACtB,MAAM,WAAW,GAAG,IAAI,CAAC,gBAAgB,CAAC,gBAAgB,CAAC,CAAC;gBAE5D,qBAAqB;gBACrB,OAAO,IAAI,qCAAqC,WAAW,WAAW,CAAC;gBACvE,IAAI,cAAc,EAAE,CAAC;oBACpB,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,cAAc,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;oBAClE,OAAO,IAAI,yBAAyB,IAAI,EAAE,CAAC;gBAC5C,CAAC;gBACD,OAAO,IAAI,YAAY,CAAC;gBACxB,OAAO,IAAI,2BAA2B,WAAW,YAAY,CAAC;YAC/D,CAAC;QACF,CAAC;QAED,mBAAmB;QACnB,MAAM,eAAe,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,4BAA4B,CAAC,CAAC,CAAC;QACjG,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAU,CAAC;QAE5C,eAAe,CAAC,OAAO,CAAC,CAAC,cAAc,EAAE,EAAE;YAC1C,MAAM,gBAAgB,GAAG,cAAc,CAAC,aAAa,CAAC,sBAAsB,CAAC,CAAC;YAC9E,IAAI,CAAC,gBAAgB;gBAAE,OAAO;YAE9B,MAAM,SAAS,GAAG,cAAc,CAAC,YAAY,CAAC,0BAA0B,CAAC,CAAC;YAC1E,IAAI,CAAC,SAAS,IAAI,iBAAiB,CAAC,GAAG,CAAC,SAAS,CAAC;gBAAE,OAAO;YAC3D,iBAAiB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAEjC,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,gBAAgB,EAAE;gBACnD,2CAA2C;gBAC3C,8BAA8B;gBAC9B,6CAA6C;aAC7C,CAAC,CAAC;YAEH,MAAM,WAAW,GAAG,gBAAgB,CAAC,aAAa,CAAC,eAAe,CAAC,CAAC;YACpE,MAAM,SAAS,GAAG,WAAW,EAAE,YAAY,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;YAE9D,MAAM,WAAW,GAAG,gBAAgB,CAAC,aAAa,CAAC,gBAAgB,CAAC,CAAC;YAErE,IAAI,WAAW,EAAE,CAAC;gBACjB,MAAM,WAAW,GAAG,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC;gBAEvD,IAAI,WAAW,EAAE,CAAC;oBACjB,OAAO,IAAI,yBAAyB,CAAC;oBACrC,OAAO,IAAI,uCAAuC,MAAM,WAAW,CAAC;oBACpE,IAAI,SAAS,EAAE,CAAC;wBACf,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;wBAC7D,OAAO,IAAI,iBAAiB,IAAI,EAAE,CAAC;oBACpC,CAAC;oBACD,OAAO,IAAI,UAAU,CAAC;oBACtB,OAAO,IAAI,6BAA6B,WAAW,UAAU,CAAC;oBAC9D,OAAO,IAAI,YAAY,CAAC;gBACzB,CAAC;YACF,CAAC;QACF,CAAC,CAAC,CAAC;QAEH,OAAO;YACN,OAAO,EAAE,OAAO;YAChB,WAAW,EAAE,OAAO;YACpB,gBAAgB,EAAE;gBACjB,IAAI,EAAE,OAAO;gBACb,WAAW;gBACX,UAAU,EAAE,QAAQ,CAAC,IAAI;gBACzB,KAAK,EAAE,QAAQ,CAAC,KAAK;aACrB;YACD,SAAS,EAAE;gBACV,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK;gBAC1B,MAAM,EAAE,EAAE;gBACV,IAAI,EAAE,YAAY,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,IAAI,EAAE;gBACnD,WAAW,EAAE,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC;aAC5C;SACD,CAAC;IACH,CAAC;IAEO,aAAa,CAAC,SAAkB,EAAE,SAAmB;QAC5D,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YAClC,MAAM,UAAU,GAAG,SAAS,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;YACrD,IAAI,UAAU,EAAE,CAAC;gBAChB,MAAM,IAAI,GAAG,UAAU,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;gBAC7C,IAAI,IAAI,EAAE,CAAC;oBACV,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;wBAC1B,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;oBAC1B,CAAC;yBAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;wBACzC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;wBACrD,IAAI,KAAK,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;4BACvB,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC;wBACjB,CAAC;oBACF,CAAC;gBACF,CAAC;YACF,CAAC;QACF,CAAC;QACD,OAAO,SAAS,CAAC;IAClB,CAAC;IAEO,gBAAgB,CAAC,WAAoB;QAC5C,MAAM,SAAS,GAAG,WAAW,CAAC,SAAS,CAAC,IAAI,CAAY,CAAC;QACzD,SAAS,CAAC,gBAAgB,CAAC,wDAAwD,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC;QAChH,SAAS,CAAC,gBAAgB,CAAC,8CAA8C,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC;QACtG,OAAO,IAAA,mBAAa,EAAC,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC;IACxC,CAAC;IAEO,kBAAkB;QACzB,gCAAgC;QAChC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAC1D,IAAI,QAAQ;YAAE,OAAO,QAAQ,CAAC,CAAC,CAAC,CAAC;QAEjC,8BAA8B;QAC9B,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;QACvD,MAAM,UAAU,GAAG,YAAY,EAAE,WAAW,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC9D,OAAO,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACxC,CAAC;IAEO,eAAe;QACtB,gCAAgC;QAChC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC;QACnE,IAAI,QAAQ,EAAE,CAAC;YACd,OAAO,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;QAClD,CAAC;QAED,8BAA8B;QAC9B,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC;QACvE,OAAO,UAAU,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;IAC7F,CAAC;IAGO,iBAAiB,CAAC,OAAe;QACxC,IAAI,CAAC,OAAO;YAAE,OAAO,EAAE,CAAC;QAExB,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;QACnD,OAAO,CAAC,WAAW,CAAC,IAAA,eAAS,EAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;QACvD,OAAO,OAAO,CAAC,WAAW,EAAE,IAAI,EAAE;aAChC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;aACb,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC;IAC9B,CAAC;CACD;AAnLD,0CAmLC"}
|
package/dist/extractors/grok.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.GrokExtractor = void 0;
|
|
4
4
|
const _conversation_1 = require("./_conversation");
|
|
5
|
+
const dom_1 = require("../utils/dom");
|
|
5
6
|
class GrokExtractor extends _conversation_1.ConversationExtractor {
|
|
6
7
|
constructor(document, url) {
|
|
7
8
|
super(document, url);
|
|
@@ -47,7 +48,7 @@ class GrokExtractor extends _conversation_1.ConversationExtractor {
|
|
|
47
48
|
// Remove known non-content elements like the DeepSearch artifact
|
|
48
49
|
clonedBubble.querySelector('.relative.border.border-border-l1.bg-surface-base')?.remove();
|
|
49
50
|
// Add selectors here for any other known elements to remove (e.g., buttons, toolbars within the bubble)
|
|
50
|
-
content = clonedBubble
|
|
51
|
+
content = (0, dom_1.serializeHTML)(clonedBubble);
|
|
51
52
|
// Process footnotes/links in the cleaned content
|
|
52
53
|
content = this.processFootnotes(content);
|
|
53
54
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"grok.js","sourceRoot":"","sources":["../../src/extractors/grok.ts"],"names":[],"mappings":";;;AAAA,mDAAwD;
|
|
1
|
+
{"version":3,"file":"grok.js","sourceRoot":"","sources":["../../src/extractors/grok.ts"],"names":[],"mappings":";;;AAAA,mDAAwD;AAExD,sCAA6C;AAE7C,MAAa,aAAc,SAAQ,qCAAqB;IAOvD,YAAY,QAAkB,EAAE,GAAW;QAC1C,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QAPtB,gGAAgG;QACxF,6BAAwB,GAAG,qDAAqD,CAAC;QAOxF,IAAI,CAAC,cAAc,GAAG,QAAQ,CAAC,gBAAgB,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;QAC/E,IAAI,CAAC,SAAS,GAAG,EAAE,CAAC;QACpB,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED,UAAU;QACT,OAAO,CAAC,CAAC,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC;IAChE,CAAC;IAES,eAAe;QACxB,MAAM,QAAQ,GAA0B,EAAE,CAAC;QAC3C,IAAI,CAAC,SAAS,GAAG,EAAE,CAAC;QACpB,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC;QAEzB,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,cAAc,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,QAAQ,CAAC;QAE9E,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE;YACzC,mFAAmF;YACnF,MAAM,aAAa,GAAG,SAAS,CAAC,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;YAChE,MAAM,aAAa,GAAG,SAAS,CAAC,SAAS,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;YAElE,IAAI,CAAC,aAAa,IAAI,CAAC,aAAa;gBAAE,OAAO,CAAC,0DAA0D;YAExG,MAAM,aAAa,GAAG,SAAS,CAAC,aAAa,CAAC,iBAAiB,CAAC,CAAC;YACjE,IAAI,CAAC,aAAa;gBAAE,OAAO,CAAC,8CAA8C;YAE1E,IAAI,OAAO,GAAW,EAAE,CAAC;YACzB,IAAI,IAAI,GAAW,EAAE,CAAC;YACtB,IAAI,MAAM,GAAW,EAAE,CAAC;YAExB,IAAI,aAAa,EAAE,CAAC;gBACnB,mEAAmE;gBACnE,8EAA8E;gBAC9E,OAAO,GAAG,aAAa,CAAC,WAAW,IAAI,EAAE,CAAC;gBAC1C,IAAI,GAAG,MAAM,CAAC;gBACd,MAAM,GAAG,KAAK,CAAC,CAAC,8DAA8D;YAC/E,CAAC;iBAAM,IAAI,aAAa,EAAE,CAAC;gBAC1B,IAAI,GAAG,WAAW,CAAC;gBACnB,MAAM,GAAG,MAAM,CAAC,CAAC,8DAA8D;gBAE/E,oEAAoE;gBACpE,MAAM,YAAY,GAAG,aAAa,CAAC,SAAS,CAAC,IAAI,CAAY,CAAC;gBAE9D,iEAAiE;gBACjE,YAAY,CAAC,aAAa,CAAC,mDAAmD,CAAC,EAAE,MAAM,EAAE,CAAC;gBAC1F,wGAAwG;gBAExG,OAAO,GAAG,IAAA,mBAAa,EAAC,YAAY,CAAC,CAAC;gBAEtC,iDAAiD;gBACjD,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;YAC1C,CAAC;YAED,IAAI,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;gBACpB,QAAQ,CAAC,IAAI,CAAC;oBACb,MAAM,EAAE,MAAM;oBACd,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE;oBACvB,QAAQ,EAAE;wBACT,IAAI,EAAE,IAAI;qBACV;iBACD,CAAC,CAAC;YACJ,CAAC;QACF,CAAC,CAAC,CAAC;QAEH,OAAO,QAAQ,CAAC;IACjB,CAAC;IAES,YAAY;QACrB,OAAO,IAAI,CAAC,SAAS,CAAC;IACvB,CAAC;IAES,WAAW;QACpB,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC9B,MAAM,YAAY,GAAG,IAAI,CAAC,cAAc,EAAE,MAAM,IAAI,CAAC,CAAC;QAEtD,OAAO;YACN,KAAK;YACL,IAAI,EAAE,MAAM;YACZ,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,YAAY,EAAE,YAAY,EAAE,sBAAsB;YAClD,WAAW,EAAE,0BAA0B,YAAY,WAAW;SAC9D,CAAC;IACH,CAAC;IAEO,QAAQ;QACf,kDAAkD;QAClD,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC;QAC9C,IAAI,SAAS,IAAI,SAAS,KAAK,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC5E,qCAAqC;YACrC,OAAO,SAAS,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACpD,CAAC;QAED,wEAAwE;QACxE,2CAA2C;QAC3C,MAAM,kBAAkB,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,IAAI,CAAC,wBAAwB,YAAY,CAAC,CAAC;QACrG,IAAI,kBAAkB,EAAE,CAAC;YACxB,MAAM,aAAa,GAAG,kBAAkB,CAAC,aAAa,CAAC,iBAAiB,CAAC,CAAC;YAC1E,IAAI,aAAa,EAAE,CAAC;gBACnB,MAAM,IAAI,GAAG,aAAa,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;gBACrD,4CAA4C;gBAC5C,OAAO,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;YAC5D,CAAC;QACF,CAAC;QAED,OAAO,mBAAmB,CAAC,CAAC,mBAAmB;IAChD,CAAC;IAEO,gBAAgB,CAAC,OAAe;QACvC,qDAAqD;QACrD,MAAM,WAAW,GAAG,qDAAqD,CAAC,CAAC,wBAAwB;QAEnG,OAAO,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE;YAC3D,kFAAkF;YACnF,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,eAAe,CAAC,EAAE,CAAC;gBAChE,OAAO,KAAK,CAAC;YACd,CAAC;YAED,oDAAoD;YACpD,IAAI,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC;YACzD,IAAI,aAAqB,CAAC;YAE1B,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACf,6CAA6C;gBAC7C,IAAI,CAAC,eAAe,EAAE,CAAC;gBACvB,aAAa,GAAG,IAAI,CAAC,eAAe,CAAC;gBAErC,IAAI,UAAU,GAAG,GAAG,CAAC,CAAC,uCAAuC;gBAC7D,IAAI,CAAC;oBACJ,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;oBAC3D,UAAU,GAAG,YAAY,GAAG,+CAA+C,MAAM,MAAM,CAAC;gBACzF,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACZ,uDAAuD;oBACvD,UAAU,GAAG,YAAY,GAAG,+CAA+C,GAAG,MAAM,CAAC;oBACrF,OAAO,CAAC,IAAI,CAAC,oDAAoD,GAAG,EAAE,CAAC,CAAC;gBACzE,CAAC;gBAED,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;oBACnB,GAAG;oBACH,IAAI,EAAE,UAAU,CAAC,+BAA+B;iBAChD,CAAC,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACP,kDAAkD;gBAClD,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,GAAG,KAAK,GAAG,CAAC,GAAG,CAAC,CAAC;YACpE,CAAC;YAED,kEAAkE;YAClE,0FAA0F;YAC1F,OAAO,GAAG,QAAQ,kBAAkB,aAAa,uCAAuC,aAAa,2BAA2B,aAAa,YAAY,CAAC;QAC3J,CAAC,CAAC,CAAC;IACJ,CAAC;CACD;AA/JD,sCA+JC"}
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.HackerNewsExtractor = void 0;
|
|
4
4
|
const _base_1 = require("./_base");
|
|
5
|
+
const dom_1 = require("../utils/dom");
|
|
5
6
|
class HackerNewsExtractor extends _base_1.BaseExtractor {
|
|
6
7
|
constructor(document, url) {
|
|
7
8
|
super(document, url);
|
|
@@ -67,7 +68,8 @@ class HackerNewsExtractor extends _base_1.BaseExtractor {
|
|
|
67
68
|
// If this is a comment page, use the comment as the main content
|
|
68
69
|
if (this.isCommentPage && this.mainComment) {
|
|
69
70
|
const author = this.mainComment.querySelector('.hnuser')?.textContent || '[deleted]';
|
|
70
|
-
const
|
|
71
|
+
const commtext = this.mainComment.querySelector('.commtext');
|
|
72
|
+
const commentText = commtext ? (0, dom_1.serializeHTML)(commtext) : '';
|
|
71
73
|
const timeElement = this.mainComment.querySelector('.age');
|
|
72
74
|
const timestamp = timeElement?.getAttribute('title') || '';
|
|
73
75
|
const date = timestamp.split('T')[0] || '';
|
|
@@ -95,7 +97,7 @@ class HackerNewsExtractor extends _base_1.BaseExtractor {
|
|
|
95
97
|
}
|
|
96
98
|
const text = this.mainPost.querySelector('.toptext');
|
|
97
99
|
if (text) {
|
|
98
|
-
content += `<div class="post-text">${text
|
|
100
|
+
content += `<div class="post-text">${(0, dom_1.serializeHTML)(text)}</div>`;
|
|
99
101
|
}
|
|
100
102
|
return content;
|
|
101
103
|
}
|
|
@@ -158,7 +160,7 @@ class HackerNewsExtractor extends _base_1.BaseExtractor {
|
|
|
158
160
|
<a href="${commentUrl}" class="comment-link">${date}</a>
|
|
159
161
|
${points ? ` • <span class="comment-points">${points}</span>` : ''}
|
|
160
162
|
</div>
|
|
161
|
-
<div class="comment-content">${commentText
|
|
163
|
+
<div class="comment-content">${(0, dom_1.serializeHTML)(commentText)}</div>
|
|
162
164
|
</div>`;
|
|
163
165
|
currentDepth = depth;
|
|
164
166
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hackernews.js","sourceRoot":"","sources":["../../src/extractors/hackernews.ts"],"names":[],"mappings":";;;AAAA,mCAAwC;
|
|
1
|
+
{"version":3,"file":"hackernews.js","sourceRoot":"","sources":["../../src/extractors/hackernews.ts"],"names":[],"mappings":";;;AAAA,mCAAwC;AAExC,sCAA6C;AAE7C,MAAa,mBAAoB,SAAQ,qBAAa;IAKrD,YAAY,QAAkB,EAAE,GAAW;QAC1C,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QACrB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;QACnD,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC9C,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IACvE,CAAC;IAEO,iBAAiB;QACxB,kFAAkF;QAClF,OAAO,CAAC,CAAC,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,yBAAyB,CAAC,CAAC;IAClE,CAAC;IAEO,eAAe;QACtB,uDAAuD;QACvD,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;QACzD,OAAO,OAAO,IAAI,IAAI,CAAC;IACxB,CAAC;IAED,UAAU;QACT,OAAO,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC;IACxB,CAAC;IAED,OAAO;QACN,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QAC1C,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,EAAE,CAAC;QAExC,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;QAClE,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;QACtC,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QACxC,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC7C,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAErC,OAAO;YACN,OAAO,EAAE,WAAW;YACpB,WAAW,EAAE,WAAW;YACxB,gBAAgB,EAAE;gBACjB,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE;gBACxB,UAAU;aACV;YACD,SAAS,EAAE;gBACV,KAAK,EAAE,SAAS;gBAChB,MAAM,EAAE,UAAU;gBAClB,IAAI,EAAE,aAAa;gBACnB,WAAW;gBACX,SAAS;aACT;SACD,CAAC;IACH,CAAC;IAEO,iBAAiB,CAAC,WAAmB,EAAE,QAAgB;QAC9D,OAAO;;;OAGF,WAAW;;MAEZ,QAAQ,CAAC,CAAC,CAAC;;;;QAIT,QAAQ;;KAEX,CAAC,CAAC,CAAC,EAAE;;GAEP,CAAC,IAAI,EAAE,CAAC;IACV,CAAC;IAEO,cAAc;QACrB,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,OAAO,EAAE,CAAC;QAE9B,iEAAiE;QACjE,IAAI,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YAC5C,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,aAAa,CAAC,SAAS,CAAC,EAAE,WAAW,IAAI,WAAW,CAAC;YACrF,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;YAC7D,MAAM,WAAW,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAA,mBAAa,EAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5D,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YAC3D,MAAM,SAAS,GAAG,WAAW,EAAE,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YAC3D,MAAM,IAAI,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC3C,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YACnF,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,yBAAyB,CAAC,EAAE,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YAErG,OAAO;;;6CAGmC,MAAM;mCAChB,IAAI;QAC/B,MAAM,CAAC,CAAC,CAAC,mCAAmC,MAAM,SAAS,CAAC,CAAC,CAAC,EAAE;QAChE,SAAS,CAAC,CAAC,CAAC,4CAA4C,SAAS,kCAAkC,CAAC,CAAC,CAAC,EAAE;;oCAE5E,WAAW;;IAE3C,CAAC,IAAI,EAAE,CAAC;QACV,CAAC;QAED,wCAAwC;QACxC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;QAC1D,MAAM,MAAM,GAAG,QAAQ,EAAE,kBAAkB,CAAC;QAC5C,MAAM,GAAG,GAAG,QAAQ,EAAE,aAAa,CAAC,cAAc,CAAC,EAAE,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QAEhF,IAAI,OAAO,GAAG,EAAE,CAAC;QACjB,IAAI,GAAG,EAAE,CAAC;YACT,OAAO,IAAI,eAAe,GAAG,qBAAqB,GAAG,UAAU,CAAC;QACjE,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;QACrD,IAAI,IAAI,EAAE,CAAC;YACV,OAAO,IAAI,0BAA0B,IAAA,mBAAa,EAAC,IAAI,CAAC,QAAQ,CAAC;QAClE,CAAC;QAED,OAAO,OAAO,CAAC;IAChB,CAAC;IAEO,eAAe;QACtB,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,CAAC;QACxE,OAAO,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;IACvC,CAAC;IAEO,eAAe,CAAC,QAAmB;QAC1C,IAAI,IAAI,GAAG,EAAE,CAAC;QACd,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;QACvC,IAAI,YAAY,GAAG,CAAC,CAAC,CAAC;QACtB,IAAI,eAAe,GAAa,EAAE,CAAC;QAEnC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAChC,MAAM,EAAE,GAAG,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YACtC,IAAI,CAAC,EAAE,IAAI,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC;gBAAE,SAAS;YAC1C,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAErB,MAAM,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE,YAAY,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC;YAC/E,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;YACpC,MAAM,WAAW,GAAG,OAAO,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;YACvD,MAAM,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,SAAS,CAAC,EAAE,WAAW,IAAI,WAAW,CAAC;YAC5E,MAAM,WAAW,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YAClD,MAAM,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YAE1E,IAAI,CAAC,WAAW;gBAAE,SAAS;YAE3B,sBAAsB;YACtB,MAAM,UAAU,GAAG,wCAAwC,EAAE,EAAE,CAAC;YAEhE,0EAA0E;YAC1E,MAAM,SAAS,GAAG,WAAW,EAAE,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YAC3D,MAAM,IAAI,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAE3C,yEAAyE;YACzE,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;gBACjB,OAAO,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACnC,IAAI,IAAI,eAAe,CAAC;oBACxB,eAAe,CAAC,GAAG,EAAE,CAAC;gBACvB,CAAC;gBACD,IAAI,IAAI,cAAc,CAAC;gBACvB,eAAe,GAAG,CAAC,CAAC,CAAC,CAAC;gBACtB,YAAY,GAAG,CAAC,CAAC;YAClB,CAAC;YACD,sBAAsB;iBACjB,CAAC;gBACL,mCAAmC;gBACnC,IAAI,KAAK,GAAG,YAAY,EAAE,CAAC;oBAC1B,OAAO,eAAe,CAAC,MAAM,GAAG,CAAC,IAAI,eAAe,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,KAAK,EAAE,CAAC;wBAC3F,IAAI,IAAI,eAAe,CAAC;wBACxB,eAAe,CAAC,GAAG,EAAE,CAAC;oBACvB,CAAC;gBACF,CAAC;gBACD,wBAAwB;qBACnB,IAAI,KAAK,GAAG,YAAY,EAAE,CAAC;oBAC/B,IAAI,IAAI,cAAc,CAAC;oBACvB,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC7B,CAAC;gBACD,mEAAmE;YACpE,CAAC;YAED,IAAI,IAAI;;yCAE8B,MAAM;aAClC,UAAU,0BAA0B,IAAI;IACjD,MAAM,CAAC,CAAC,CAAC,mCAAmC,MAAM,SAAS,CAAC,CAAC,CAAC,EAAE;;gCAEpC,IAAA,mBAAa,EAAC,WAAW,CAAC;OACnD,CAAC;YAEL,YAAY,GAAG,KAAK,CAAC;QACtB,CAAC;QAED,kCAAkC;QAClC,OAAO,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnC,IAAI,IAAI,eAAe,CAAC;YACxB,eAAe,CAAC,GAAG,EAAE,CAAC;QACvB,CAAC;QAED,OAAO,IAAI,CAAC;IACb,CAAC;IAEO,SAAS;QAChB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QACzC,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACzB,CAAC;IAEO,YAAY;QACnB,IAAI,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YAC5C,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,aAAa,CAAC,SAAS,CAAC,EAAE,WAAW,IAAI,WAAW,CAAC;YACrF,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,aAAa,CAAC,WAAW,CAAC,EAAE,WAAW,IAAI,EAAE,CAAC;YACnF,8CAA8C;YAC9C,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACzF,OAAO,cAAc,MAAM,KAAK,OAAO,EAAE,CAAC;QAC3C,CAAC;QACD,OAAO,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,YAAY,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IAC9E,CAAC;IAEO,aAAa;QACpB,OAAO,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,SAAS,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IAC3E,CAAC;IAEO,iBAAiB;QACxB,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QACpC,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACxB,OAAO,cAAc,MAAM,iBAAiB,CAAC;QAC9C,CAAC;QACD,OAAO,GAAG,KAAK,SAAS,MAAM,iBAAiB,CAAC;IACjD,CAAC;IAEO,WAAW;QAClB,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,OAAO,EAAE,CAAC;QAE9B,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;QACxD,MAAM,SAAS,GAAG,WAAW,EAAE,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAC3D,OAAO,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACtC,CAAC;CACD;AAxOD,kDAwOC"}
|
|
@@ -2,9 +2,14 @@ import { BaseExtractor } from './_base';
|
|
|
2
2
|
import { ExtractorResult } from '../types/extractors';
|
|
3
3
|
export declare class RedditExtractor extends BaseExtractor {
|
|
4
4
|
private shredditPost;
|
|
5
|
+
private isOldReddit;
|
|
5
6
|
constructor(document: Document, url: string);
|
|
6
7
|
canExtract(): boolean;
|
|
8
|
+
canExtractAsync(): boolean;
|
|
9
|
+
private isCommentsPage;
|
|
10
|
+
extractAsync(): Promise<ExtractorResult>;
|
|
7
11
|
extract(): ExtractorResult;
|
|
12
|
+
private extractOldReddit;
|
|
8
13
|
private getPostContent;
|
|
9
14
|
private createContentHtml;
|
|
10
15
|
private extractComments;
|
|
@@ -12,5 +17,7 @@ export declare class RedditExtractor extends BaseExtractor {
|
|
|
12
17
|
private getSubreddit;
|
|
13
18
|
private getPostAuthor;
|
|
14
19
|
private createDescription;
|
|
20
|
+
private processOldRedditComments;
|
|
21
|
+
private renderOldRedditComment;
|
|
15
22
|
private processComments;
|
|
16
23
|
}
|
|
@@ -2,15 +2,57 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.RedditExtractor = void 0;
|
|
4
4
|
const _base_1 = require("./_base");
|
|
5
|
+
const dom_1 = require("../utils/dom");
|
|
5
6
|
class RedditExtractor extends _base_1.BaseExtractor {
|
|
6
7
|
constructor(document, url) {
|
|
7
8
|
super(document, url);
|
|
8
9
|
this.shredditPost = document.querySelector('shreddit-post');
|
|
10
|
+
this.isOldReddit = !!document.querySelector('.thing.link');
|
|
9
11
|
}
|
|
10
12
|
canExtract() {
|
|
11
|
-
return !!this.shredditPost;
|
|
13
|
+
return !!this.shredditPost || this.isOldReddit;
|
|
14
|
+
}
|
|
15
|
+
canExtractAsync() {
|
|
16
|
+
// For new reddit comment pages, extract() returns empty content
|
|
17
|
+
// when shreddit-comment elements are missing (server-side fetch),
|
|
18
|
+
// causing parseAsync() to fall through to this async path.
|
|
19
|
+
return this.isCommentsPage() && !this.isOldReddit;
|
|
20
|
+
}
|
|
21
|
+
isCommentsPage() {
|
|
22
|
+
return /\/r\/.+\/comments\//.test(this.url);
|
|
23
|
+
}
|
|
24
|
+
async extractAsync() {
|
|
25
|
+
// Convert URL to old.reddit.com
|
|
26
|
+
const oldUrl = new URL(this.url);
|
|
27
|
+
oldUrl.hostname = 'old.reddit.com';
|
|
28
|
+
const response = await fetch(oldUrl.toString(), {
|
|
29
|
+
headers: {
|
|
30
|
+
'User-Agent': 'Mozilla/5.0 (compatible; Defuddle/1.0)',
|
|
31
|
+
},
|
|
32
|
+
});
|
|
33
|
+
if (!response.ok) {
|
|
34
|
+
throw new Error(`Failed to fetch old.reddit.com: ${response.status}`);
|
|
35
|
+
}
|
|
36
|
+
const html = await response.text();
|
|
37
|
+
const Parser = this.document.defaultView?.DOMParser ?? (typeof DOMParser !== 'undefined' ? DOMParser : null);
|
|
38
|
+
if (!Parser) {
|
|
39
|
+
throw new Error('DOMParser is not available in this environment');
|
|
40
|
+
}
|
|
41
|
+
const doc = new Parser().parseFromString(html, 'text/html');
|
|
42
|
+
return this.extractOldReddit(doc);
|
|
12
43
|
}
|
|
13
44
|
extract() {
|
|
45
|
+
if (this.isOldReddit) {
|
|
46
|
+
return this.extractOldReddit(this.document);
|
|
47
|
+
}
|
|
48
|
+
// New reddit server-side HTML includes shreddit-post but not
|
|
49
|
+
// shreddit-comment elements (those require JS). Return empty
|
|
50
|
+
// so parseAsync() falls through to extractAsync() which fetches
|
|
51
|
+
// old.reddit.com with full content.
|
|
52
|
+
const hasComments = this.document.querySelectorAll('shreddit-comment').length > 0;
|
|
53
|
+
if (this.isCommentsPage() && !hasComments) {
|
|
54
|
+
return { content: '', contentHtml: '' };
|
|
55
|
+
}
|
|
14
56
|
const postContent = this.getPostContent();
|
|
15
57
|
const comments = this.extractComments();
|
|
16
58
|
const contentHtml = this.createContentHtml(postContent, comments);
|
|
@@ -34,8 +76,36 @@ class RedditExtractor extends _base_1.BaseExtractor {
|
|
|
34
76
|
}
|
|
35
77
|
};
|
|
36
78
|
}
|
|
79
|
+
extractOldReddit(root) {
|
|
80
|
+
const thingLink = root.querySelector('.thing.link');
|
|
81
|
+
const postTitle = thingLink?.querySelector('a.title')?.textContent?.trim() || '';
|
|
82
|
+
const postAuthor = thingLink?.getAttribute('data-author') || '';
|
|
83
|
+
const subreddit = thingLink?.getAttribute('data-subreddit') || '';
|
|
84
|
+
const postBodyEl = thingLink?.querySelector('.usertext-body .md');
|
|
85
|
+
const postBody = postBodyEl ? (0, dom_1.serializeHTML)(postBodyEl) : '';
|
|
86
|
+
const commentArea = root.querySelector('.commentarea .sitetable');
|
|
87
|
+
const comments = commentArea ? this.processOldRedditComments(commentArea) : '';
|
|
88
|
+
const contentHtml = this.createContentHtml(postBody, comments);
|
|
89
|
+
const description = this.createDescription(postBody);
|
|
90
|
+
return {
|
|
91
|
+
content: contentHtml,
|
|
92
|
+
contentHtml: contentHtml,
|
|
93
|
+
extractedContent: {
|
|
94
|
+
postId: this.getPostId(),
|
|
95
|
+
subreddit,
|
|
96
|
+
postAuthor,
|
|
97
|
+
},
|
|
98
|
+
variables: {
|
|
99
|
+
title: postTitle,
|
|
100
|
+
author: postAuthor,
|
|
101
|
+
site: `r/${subreddit}`,
|
|
102
|
+
description,
|
|
103
|
+
}
|
|
104
|
+
};
|
|
105
|
+
}
|
|
37
106
|
getPostContent() {
|
|
38
|
-
const
|
|
107
|
+
const textBodyEl = this.shredditPost?.querySelector('[slot="text-body"]');
|
|
108
|
+
const textBody = textBodyEl ? (0, dom_1.serializeHTML)(textBodyEl) : '';
|
|
39
109
|
const mediaBody = this.shredditPost?.querySelector('#post-image')?.outerHTML || '';
|
|
40
110
|
return textBody + mediaBody;
|
|
41
111
|
}
|
|
@@ -74,11 +144,44 @@ class RedditExtractor extends _base_1.BaseExtractor {
|
|
|
74
144
|
if (!postContent)
|
|
75
145
|
return '';
|
|
76
146
|
const tempDiv = this.document.createElement('div');
|
|
77
|
-
tempDiv.
|
|
147
|
+
tempDiv.appendChild((0, dom_1.parseHTML)(this.document, postContent));
|
|
78
148
|
return tempDiv.textContent?.trim()
|
|
79
149
|
.slice(0, 140)
|
|
80
150
|
.replace(/\s+/g, ' ') || '';
|
|
81
151
|
}
|
|
152
|
+
processOldRedditComments(container) {
|
|
153
|
+
const topLevelComments = Array.from(container.querySelectorAll(':scope > .thing.comment'));
|
|
154
|
+
return topLevelComments.map(comment => this.renderOldRedditComment(comment)).join('');
|
|
155
|
+
}
|
|
156
|
+
renderOldRedditComment(comment) {
|
|
157
|
+
const author = comment.getAttribute('data-author') || '';
|
|
158
|
+
const permalink = comment.getAttribute('data-permalink') || '';
|
|
159
|
+
const score = comment.querySelector('.entry .tagline .score.unvoted')?.textContent?.trim() || '';
|
|
160
|
+
const timeEl = comment.querySelector('.entry .tagline time[datetime]');
|
|
161
|
+
const datetime = timeEl?.getAttribute('datetime') || '';
|
|
162
|
+
const date = datetime ? new Date(datetime).toISOString().split('T')[0] : '';
|
|
163
|
+
const bodyEl = comment.querySelector('.entry .usertext-body .md');
|
|
164
|
+
const body = bodyEl ? (0, dom_1.serializeHTML)(bodyEl) : '';
|
|
165
|
+
let html = '<blockquote>';
|
|
166
|
+
html += `<div class="comment">
|
|
167
|
+
<div class="comment-metadata">
|
|
168
|
+
<span class="comment-author"><strong>${author}</strong></span> •
|
|
169
|
+
<a href="https://reddit.com${permalink}" class="comment-link">${score}</a> •
|
|
170
|
+
<span class="comment-date">${date}</span>
|
|
171
|
+
</div>
|
|
172
|
+
<div class="comment-content">${body}</div>
|
|
173
|
+
</div>`;
|
|
174
|
+
// Recurse into child comments
|
|
175
|
+
const childContainer = comment.querySelector('.child > .sitetable');
|
|
176
|
+
if (childContainer) {
|
|
177
|
+
const children = Array.from(childContainer.querySelectorAll(':scope > .thing.comment'));
|
|
178
|
+
for (const child of children) {
|
|
179
|
+
html += this.renderOldRedditComment(child);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
html += '</blockquote>';
|
|
183
|
+
return html;
|
|
184
|
+
}
|
|
82
185
|
processComments(comments) {
|
|
83
186
|
let html = '';
|
|
84
187
|
let currentDepth = -1;
|
|
@@ -88,7 +191,8 @@ class RedditExtractor extends _base_1.BaseExtractor {
|
|
|
88
191
|
const author = comment.getAttribute('author') || '';
|
|
89
192
|
const score = comment.getAttribute('score') || '0';
|
|
90
193
|
const permalink = comment.getAttribute('permalink') || '';
|
|
91
|
-
const
|
|
194
|
+
const commentEl = comment.querySelector('[slot="comment"]');
|
|
195
|
+
const content = commentEl ? (0, dom_1.serializeHTML)(commentEl) : '';
|
|
92
196
|
// Get timestamp from faceplate-timeago element
|
|
93
197
|
const timeElement = comment.querySelector('faceplate-timeago');
|
|
94
198
|
const timestamp = timeElement?.getAttribute('ts') || '';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"reddit.js","sourceRoot":"","sources":["../../src/extractors/reddit.ts"],"names":[],"mappings":";;;AAAA,mCAAwC;
|
|
1
|
+
{"version":3,"file":"reddit.js","sourceRoot":"","sources":["../../src/extractors/reddit.ts"],"names":[],"mappings":";;;AAAA,mCAAwC;AAExC,sCAAwD;AAExD,MAAa,eAAgB,SAAQ,qBAAa;IAIjD,YAAY,QAAkB,EAAE,GAAW;QAC1C,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QACrB,IAAI,CAAC,YAAY,GAAG,QAAQ,CAAC,aAAa,CAAC,eAAe,CAAC,CAAC;QAC5D,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC;IAC5D,CAAC;IAED,UAAU;QACT,OAAO,CAAC,CAAC,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,WAAW,CAAC;IAChD,CAAC;IAED,eAAe;QACd,gEAAgE;QAChE,kEAAkE;QAClE,2DAA2D;QAC3D,OAAO,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC;IACnD,CAAC;IAEO,cAAc;QACrB,OAAO,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,YAAY;QACjB,gCAAgC;QAChC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjC,MAAM,CAAC,QAAQ,GAAG,gBAAgB,CAAC;QAEnC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE;YAC/C,OAAO,EAAE;gBACR,YAAY,EAAE,wCAAwC;aACtD;SACD,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CAAC,mCAAmC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QACvE,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,SAAS,IAAI,CAAC,OAAO,SAAS,KAAK,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC7G,IAAI,CAAC,MAAM,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QACnE,CAAC;QACD,MAAM,GAAG,GAAG,IAAI,MAAM,EAAE,CAAC,eAAe,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;QAE5D,OAAO,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;IACnC,CAAC;IAED,OAAO;QACN,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC7C,CAAC;QAED,6DAA6D;QAC7D,6DAA6D;QAC7D,gEAAgE;QAChE,oCAAoC;QACpC,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;QAClF,IAAI,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC;YAC3C,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC;QACzC,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QAC1C,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,EAAE,CAAC;QAExC,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;QAClE,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAC/E,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;QACtC,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QACxC,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;QAExD,OAAO;YACN,OAAO,EAAE,WAAW;YACpB,WAAW,EAAE,WAAW;YACxB,gBAAgB,EAAE;gBACjB,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE;gBACxB,SAAS;gBACT,UAAU;aACV;YACD,SAAS,EAAE;gBACV,KAAK,EAAE,SAAS;gBAChB,MAAM,EAAE,UAAU;gBAClB,IAAI,EAAE,KAAK,SAAS,EAAE;gBACtB,WAAW;aACX;SACD,CAAC;IACH,CAAC;IAEO,gBAAgB,CAAC,IAAwB;QAChD,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC;QACpD,MAAM,SAAS,GAAG,SAAS,EAAE,aAAa,CAAC,SAAS,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QACjF,MAAM,UAAU,GAAG,SAAS,EAAE,YAAY,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;QAChE,MAAM,SAAS,GAAG,SAAS,EAAE,YAAY,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC;QAClE,MAAM,UAAU,GAAG,SAAS,EAAE,aAAa,CAAC,oBAAoB,CAAC,CAAC;QAClE,MAAM,QAAQ,GAAG,UAAU,CAAC,CAAC,CAAC,IAAA,mBAAa,EAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAE7D,MAAM,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,yBAAyB,CAAC,CAAC;QAClE,MAAM,QAAQ,GAAG,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,wBAAwB,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAE/E,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAC/D,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAErD,OAAO;YACN,OAAO,EAAE,WAAW;YACpB,WAAW,EAAE,WAAW;YACxB,gBAAgB,EAAE;gBACjB,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE;gBACxB,SAAS;gBACT,UAAU;aACV;YACD,SAAS,EAAE;gBACV,KAAK,EAAE,SAAS;gBAChB,MAAM,EAAE,UAAU;gBAClB,IAAI,EAAE,KAAK,SAAS,EAAE;gBACtB,WAAW;aACX;SACD,CAAC;IACH,CAAC;IAEO,cAAc;QACrB,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,EAAE,aAAa,CAAC,oBAAoB,CAAC,CAAC;QAC1E,MAAM,QAAQ,GAAG,UAAU,CAAC,CAAC,CAAC,IAAA,mBAAa,EAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7D,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,EAAE,aAAa,CAAC,aAAa,CAAC,EAAE,SAAS,IAAI,EAAE,CAAC;QAEnF,OAAO,QAAQ,GAAG,SAAS,CAAC;IAC7B,CAAC;IAEO,iBAAiB,CAAC,WAAmB,EAAE,QAAgB;QAC9D,OAAO;;;OAGF,WAAW;;;KAGb,QAAQ,CAAC,CAAC,CAAC;;;;OAIT,QAAQ;;IAEX,CAAC,CAAC,CAAC,EAAE;GACN,CAAC,IAAI,EAAE,CAAC;IACV,CAAC;IAEO,eAAe;QACtB,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,CAAC,CAAC;QAChF,OAAO,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;IACvC,CAAC;IAEO,SAAS;QAChB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;QACzD,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACzB,CAAC;IAEO,YAAY;QACnB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;QAC7C,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACzB,CAAC;IAEO,aAAa;QACpB,OAAO,IAAI,CAAC,YAAY,EAAE,YAAY,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;IACxD,CAAC;IAEO,iBAAiB,CAAC,WAAmB;QAC5C,IAAI,CAAC,WAAW;YAAE,OAAO,EAAE,CAAC;QAE5B,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;QACnD,OAAO,CAAC,WAAW,CAAC,IAAA,eAAS,EAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC,CAAC;QAC3D,OAAO,OAAO,CAAC,WAAW,EAAE,IAAI,EAAE;aAChC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;aACb,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC;IAC9B,CAAC;IAEO,wBAAwB,CAAC,SAAkB;QAClD,MAAM,gBAAgB,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,yBAAyB,CAAC,CAAC,CAAC;QAC3F,OAAO,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,IAAI,CAAC,sBAAsB,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACvF,CAAC;IAEO,sBAAsB,CAAC,OAAgB;QAC9C,MAAM,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;QACzD,MAAM,SAAS,GAAG,OAAO,CAAC,YAAY,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC;QAC/D,MAAM,KAAK,GAAG,OAAO,CAAC,aAAa,CAAC,gCAAgC,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QACjG,MAAM,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,gCAAgC,CAAC,CAAC;QACvE,MAAM,QAAQ,GAAG,MAAM,EAAE,YAAY,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;QACxD,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5E,MAAM,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,2BAA2B,CAAC,CAAC;QAClE,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,IAAA,mBAAa,EAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAEjD,IAAI,IAAI,GAAG,cAAc,CAAC;QAC1B,IAAI,IAAI;;yCAE+B,MAAM;+BAChB,SAAS,0BAA0B,KAAK;+BACxC,IAAI;;gCAEH,IAAI;OAC7B,CAAC;QAEN,8BAA8B;QAC9B,MAAM,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC,qBAAqB,CAAC,CAAC;QACpE,IAAI,cAAc,EAAE,CAAC;YACpB,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,gBAAgB,CAAC,yBAAyB,CAAC,CAAC,CAAC;YACxF,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;gBAC9B,IAAI,IAAI,IAAI,CAAC,sBAAsB,CAAC,KAAK,CAAC,CAAC;YAC5C,CAAC;QACF,CAAC;QAED,IAAI,IAAI,eAAe,CAAC;QACxB,OAAO,IAAI,CAAC;IACb,CAAC;IAEO,eAAe,CAAC,QAAmB;QAC1C,IAAI,IAAI,GAAG,EAAE,CAAC;QACd,IAAI,YAAY,GAAG,CAAC,CAAC,CAAC;QACtB,IAAI,eAAe,GAAa,EAAE,CAAC,CAAC,+CAA+C;QAEnF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAChC,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,CAAC;YAC7D,MAAM,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YACpD,MAAM,KAAK,GAAG,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC;YACnD,MAAM,SAAS,GAAG,OAAO,CAAC,YAAY,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;YAC1D,MAAM,SAAS,GAAG,OAAO,CAAC,aAAa,CAAC,kBAAkB,CAAC,CAAC;YAC5D,MAAM,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,IAAA,mBAAa,EAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAE1D,+CAA+C;YAC/C,MAAM,WAAW,GAAG,OAAO,CAAC,aAAa,CAAC,mBAAmB,CAAC,CAAC;YAC/D,MAAM,SAAS,GAAG,WAAW,EAAE,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YACxD,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAE9E,yEAAyE;YACzE,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;gBACjB,6BAA6B;gBAC7B,OAAO,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACnC,IAAI,IAAI,eAAe,CAAC;oBACxB,eAAe,CAAC,GAAG,EAAE,CAAC;gBACvB,CAAC;gBACD,IAAI,IAAI,cAAc,CAAC;gBACvB,eAAe,GAAG,CAAC,CAAC,CAAC,CAAC;gBACtB,YAAY,GAAG,CAAC,CAAC;YAClB,CAAC;YACD,sBAAsB;iBACjB,CAAC;gBACL,mCAAmC;gBACnC,IAAI,KAAK,GAAG,YAAY,EAAE,CAAC;oBAC1B,qDAAqD;oBACrD,OAAO,eAAe,CAAC,MAAM,GAAG,CAAC,IAAI,eAAe,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,KAAK,EAAE,CAAC;wBAC3F,IAAI,IAAI,eAAe,CAAC;wBACxB,eAAe,CAAC,GAAG,EAAE,CAAC;oBACvB,CAAC;gBACF,CAAC;gBACD,wBAAwB;qBACnB,IAAI,KAAK,GAAG,YAAY,EAAE,CAAC;oBAC/B,IAAI,IAAI,cAAc,CAAC;oBACvB,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC7B,CAAC;gBACD,mEAAmE;YACpE,CAAC;YAED,IAAI,IAAI;;yCAE8B,MAAM;+BAChB,SAAS,0BAA0B,KAAK;+BACxC,IAAI;;gCAEH,OAAO;OAChC,CAAC;YAEL,YAAY,GAAG,KAAK,CAAC;QACtB,CAAC;QAED,kCAAkC;QAClC,OAAO,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnC,IAAI,IAAI,eAAe,CAAC;YACxB,eAAe,CAAC,GAAG,EAAE,CAAC;QACvB,CAAC;QAED,OAAO,IAAI,CAAC;IACb,CAAC;CACD;AAxRD,0CAwRC"}
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.TwitterExtractor = void 0;
|
|
4
4
|
const _base_1 = require("./_base");
|
|
5
|
+
const dom_1 = require("../utils/dom");
|
|
5
6
|
class TwitterExtractor extends _base_1.BaseExtractor {
|
|
6
7
|
constructor(document, url) {
|
|
7
8
|
super(document, url);
|
|
@@ -73,7 +74,7 @@ class TwitterExtractor extends _base_1.BaseExtractor {
|
|
|
73
74
|
return '';
|
|
74
75
|
// Create a temporary div to parse and clean the HTML
|
|
75
76
|
const tempDiv = this.document.createElement('div');
|
|
76
|
-
tempDiv.
|
|
77
|
+
tempDiv.appendChild((0, dom_1.parseHTML)(this.document, text));
|
|
77
78
|
// Convert links to plain text with @ handles
|
|
78
79
|
tempDiv.querySelectorAll('a').forEach(link => {
|
|
79
80
|
const handle = link.textContent?.trim() || '';
|
|
@@ -84,7 +85,7 @@ class TwitterExtractor extends _base_1.BaseExtractor {
|
|
|
84
85
|
element.replaceWith(...Array.from(element.childNodes));
|
|
85
86
|
});
|
|
86
87
|
// Get cleaned text and split into paragraphs
|
|
87
|
-
const cleanText = tempDiv
|
|
88
|
+
const cleanText = (0, dom_1.serializeHTML)(tempDiv);
|
|
88
89
|
const paragraphs = cleanText.split('\n')
|
|
89
90
|
.map(line => line.trim())
|
|
90
91
|
.filter(line => line);
|
|
@@ -105,7 +106,8 @@ class TwitterExtractor extends _base_1.BaseExtractor {
|
|
|
105
106
|
}
|
|
106
107
|
}
|
|
107
108
|
});
|
|
108
|
-
const
|
|
109
|
+
const tweetTextEl = tweetClone.querySelector('[data-testid="tweetText"]');
|
|
110
|
+
const tweetText = tweetTextEl ? (0, dom_1.serializeHTML)(tweetTextEl) : '';
|
|
109
111
|
const formattedText = this.formatTweetText(tweetText);
|
|
110
112
|
const images = this.extractImages(tweet);
|
|
111
113
|
// Get author info and date
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"twitter.js","sourceRoot":"","sources":["../../src/extractors/twitter.ts"],"names":[],"mappings":";;;AAAA,mCAAwC;
|
|
1
|
+
{"version":3,"file":"twitter.js","sourceRoot":"","sources":["../../src/extractors/twitter.ts"],"names":[],"mappings":";;;AAAA,mCAAwC;AAExC,sCAAwD;AAExD,MAAa,gBAAiB,SAAQ,qBAAa;IAIlD,YAAY,QAAkB,EAAE,GAAW;QAC1C,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QAJd,cAAS,GAAmB,IAAI,CAAC;QACjC,iBAAY,GAAc,EAAE,CAAC;QAKpC,mCAAmC;QACnC,MAAM,QAAQ,GAAG,QAAQ,CAAC,aAAa,CAAC,uCAAuC,CAAC,CAAC;QACjF,IAAI,CAAC,QAAQ,EAAE,CAAC;YACf,qDAAqD;YACrD,MAAM,WAAW,GAAG,QAAQ,CAAC,aAAa,CAAC,8BAA8B,CAAC,CAAC;YAC3E,IAAI,WAAW,EAAE,CAAC;gBACjB,IAAI,CAAC,SAAS,GAAG,WAAW,CAAC;YAC9B,CAAC;YACD,OAAO;QACR,CAAC;QAED,6EAA6E;QAC7E,IAAI,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,8BAA8B,CAAC,CAAC,CAAC;QACtF,MAAM,YAAY,GAAG,QAAQ,CAAC,aAAa,CAAC,aAAa,CAAC,EAAE,aAAa,CAAC;QAE1E,IAAI,YAAY,EAAE,CAAC;YAClB,wDAAwD;YACxD,MAAM,WAAW,GAAG,SAAS,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAC/C,YAAY,CAAC,uBAAuB,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,2BAA2B,CAC9E,CAAC;YACF,IAAI,WAAW,KAAK,CAAC,CAAC,EAAE,CAAC;gBACxB,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;YAC7C,CAAC;QACF,CAAC;QAED,mCAAmC;QACnC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;QACtC,IAAI,CAAC,YAAY,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACxC,CAAC;IAED,UAAU;QACT,OAAO,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;IACzB,CAAC;IAED,OAAO;QACN,MAAM,WAAW,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtD,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAEhG,MAAM,WAAW,GAAG;;;OAGf,WAAW;;MAEZ,aAAa,CAAC,CAAC,CAAC;;;QAGd,aAAa;;KAEhB,CAAC,CAAC,CAAC,EAAE;;GAEP,CAAC,IAAI,EAAE,CAAC;QAET,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;QAClC,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QAC1C,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAE3D,OAAO;YACN,OAAO,EAAE,WAAW;YACpB,WAAW,EAAE,WAAW;YACxB,gBAAgB,EAAE;gBACjB,OAAO;gBACP,WAAW;aACX;YACD,SAAS,EAAE;gBACV,KAAK,EAAE,aAAa,WAAW,EAAE;gBACjC,MAAM,EAAE,WAAW;gBACnB,IAAI,EAAE,aAAa;gBACnB,WAAW;aACX;SACD,CAAC;IACH,CAAC;IAEO,eAAe,CAAC,IAAY;QACnC,IAAI,CAAC,IAAI;YAAE,OAAO,EAAE,CAAC;QAErB,qDAAqD;QACrD,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;QACnD,OAAO,CAAC,WAAW,CAAC,IAAA,eAAS,EAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC;QAEpD,6CAA6C;QAC7C,OAAO,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE;YAC5C,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YAC9C,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;QAEH,2DAA2D;QAC3D,OAAO,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;YACvD,OAAO,CAAC,WAAW,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;QACxD,CAAC,CAAC,CAAC;QAEH,6CAA6C;QAC7C,MAAM,SAAS,GAAG,IAAA,mBAAa,EAAC,OAAO,CAAC,CAAC;QACzC,MAAM,UAAU,GAAG,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC;aACtC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;aACxB,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;QAEvB,kCAAkC;QAClC,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACtD,CAAC;IAEO,YAAY,CAAC,KAAqB;QACzC,IAAI,CAAC,KAAK;YAAE,OAAO,EAAE,CAAC;QAEtB,uCAAuC;QACvC,MAAM,UAAU,GAAG,KAAK,CAAC,SAAS,CAAC,IAAI,CAAY,CAAC;QAEpD,+BAA+B;QAC/B,UAAU,CAAC,gBAAgB,CAAC,qBAAqB,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;YAChE,IAAI,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,KAAK,KAAK,IAAI,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;gBACpE,MAAM,OAAO,GAAG,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;gBACxC,IAAI,OAAO,EAAE,CAAC;oBACb,GAAG,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;gBAC1B,CAAC;YACF,CAAC;QACF,CAAC,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,UAAU,CAAC,aAAa,CAAC,2BAA2B,CAAC,CAAC;QAC1E,MAAM,SAAS,GAAG,WAAW,CAAC,CAAC,CAAC,IAAA,mBAAa,EAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAChE,MAAM,aAAa,GAAG,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;QACtD,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;QAEzC,2BAA2B;QAC3B,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QAE7C,kCAAkC;QAClC,MAAM,WAAW,GAAG,KAAK,CAAC,aAAa,CAAC,2BAA2B,CAAC,EAAE,aAAa,CAAC,2BAA2B,CAAC,EAAE,OAAO,CAAC,2BAA2B,CAAC,CAAC;QACvJ,MAAM,aAAa,GAAG,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAExE,OAAO;;;0CAGiC,QAAQ,CAAC,QAAQ,wCAAwC,QAAQ,CAAC,MAAM;OAC3G,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,YAAY,QAAQ,CAAC,SAAS,wBAAwB,QAAQ,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE;;MAE/F,aAAa,CAAC,CAAC,CAAC,2BAA2B,aAAa,QAAQ,CAAC,CAAC,CAAC,EAAE;MACrE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;;QAEd,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;;KAEpB,CAAC,CAAC,CAAC,EAAE;MACJ,aAAa,CAAC,CAAC,CAAC;;QAEd,aAAa;;KAEhB,CAAC,CAAC,CAAC,EAAE;;GAEP,CAAC,IAAI,EAAE,CAAC;IACV,CAAC;IAEO,eAAe,CAAC,KAAc;QACrC,MAAM,WAAW,GAAG,KAAK,CAAC,aAAa,CAAC,2BAA2B,CAAC,CAAC;QACrE,IAAI,CAAC,WAAW;YAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;QAE/E,qEAAqE;QACrE,MAAM,KAAK,GAAG,WAAW,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;QAChD,IAAI,QAAQ,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QACrD,IAAI,MAAM,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAEnD,+EAA+E;QAC/E,IAAI,CAAC,QAAQ,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1B,QAAQ,GAAG,WAAW,CAAC,aAAa,CAAC,4CAA4C,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YAC9G,MAAM,GAAG,WAAW,CAAC,aAAa,CAAC,yCAAyC,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAC1G,CAAC;QAED,MAAM,SAAS,GAAG,KAAK,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;QAC9C,MAAM,QAAQ,GAAG,SAAS,EAAE,YAAY,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;QAC3D,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5E,MAAM,SAAS,GAAG,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,EAAE,IAAI,IAAI,EAAE,CAAC;QAEtD,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;IAC9C,CAAC;IAEO,aAAa,CAAC,KAAc;QACnC,0CAA0C;QAC1C,MAAM,eAAe,GAAG;YACvB,4BAA4B;YAC5B,6BAA6B;YAC7B,mBAAmB;SACnB,CAAC;QAEF,MAAM,MAAM,GAAa,EAAE,CAAC;QAE5B,4CAA4C;QAC5C,MAAM,WAAW,GAAG,KAAK,CAAC,aAAa,CAAC,2BAA2B,CAAC,EAAE,aAAa,CAAC,2BAA2B,CAAC,EAAE,OAAO,CAAC,2BAA2B,CAAC,CAAC;QAEvJ,KAAK,MAAM,QAAQ,IAAI,eAAe,EAAE,CAAC;YACxC,MAAM,QAAQ,GAAG,KAAK,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;YAElD,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;gBACtB,6CAA6C;gBAC7C,IAAI,WAAW,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBAChC,OAAO;gBACR,CAAC;gBAED,4EAA4E;gBAC5E,IAAI,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,KAAK,KAAK,IAAI,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;oBACpE,MAAM,cAAc,GAAG,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC,YAAY,EAAE,aAAa,CAAC,IAAI,EAAE,CAAC;oBAC3F,MAAM,QAAQ,GAAG,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC;oBAC5E,MAAM,CAAC,IAAI,CAAC,aAAa,cAAc,UAAU,QAAQ,MAAM,CAAC,CAAC;gBAClE,CAAC;YACF,CAAC,CAAC,CAAC;QACJ,CAAC;QAED,OAAO,MAAM,CAAC;IACf,CAAC;IAEO,UAAU;QACjB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;QAC9C,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACzB,CAAC;IAEO,cAAc;QACrB,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,EAAE,aAAa,CAAC,2BAA2B,CAAC,CAAC;QAC/E,MAAM,KAAK,GAAG,WAAW,EAAE,gBAAgB,CAAC,GAAG,CAAC,CAAC;QACjD,MAAM,MAAM,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QACrD,OAAO,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,MAAM,EAAE,CAAC;IACvD,CAAC;IAEO,iBAAiB,CAAC,KAAqB;QAC9C,IAAI,CAAC,KAAK;YAAE,OAAO,EAAE,CAAC;QAEtB,MAAM,SAAS,GAAG,KAAK,CAAC,aAAa,CAAC,2BAA2B,CAAC,EAAE,WAAW,IAAI,EAAE,CAAC;QACtF,OAAO,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC5D,CAAC;CACD;AAvOD,4CAuOC"}
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.XArticleExtractor = void 0;
|
|
4
4
|
const _base_1 = require("./_base");
|
|
5
|
+
const dom_1 = require("../utils/dom");
|
|
5
6
|
const SELECTORS = {
|
|
6
7
|
ARTICLE_CONTAINER: '[data-testid="twitterArticleRichTextView"]',
|
|
7
8
|
TITLE: '[data-testid="twitter-article-title"]',
|
|
@@ -79,7 +80,7 @@ class XArticleExtractor extends _base_1.BaseExtractor {
|
|
|
79
80
|
return '';
|
|
80
81
|
const clone = this.articleContainer.cloneNode(true);
|
|
81
82
|
this.cleanContent(clone);
|
|
82
|
-
return `<article class="x-article">${clone
|
|
83
|
+
return `<article class="x-article">${(0, dom_1.serializeHTML)(clone)}</article>`;
|
|
83
84
|
}
|
|
84
85
|
cleanContent(container) {
|
|
85
86
|
const ownerDoc = container.ownerDocument || this.document;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"x-article.js","sourceRoot":"","sources":["../../src/extractors/x-article.ts"],"names":[],"mappings":";;;AAAA,mCAAwC;
|
|
1
|
+
{"version":3,"file":"x-article.js","sourceRoot":"","sources":["../../src/extractors/x-article.ts"],"names":[],"mappings":";;;AAAA,mCAAwC;AAExC,sCAA6C;AAE7C,MAAM,SAAS,GAAG;IACjB,iBAAiB,EAAE,4CAA4C;IAC/D,KAAK,EAAE,uCAAuC;IAC9C,MAAM,EAAE,qBAAqB;IAC7B,WAAW,EAAE,uBAAuB;IACpC,aAAa,EAAE,iCAAiC;IAChD,MAAM,EAAE,gCAAgC;IACxC,gBAAgB,EAAE,qDAAqD;IACvE,UAAU,EAAE,kCAAkC;IAC9C,gBAAgB,EAAE,mBAAmB;IACrC,cAAc,EAAE,6BAA6B;IAC7C,UAAU,EAAE,2BAA2B;IACvC,SAAS,EAAE,2BAA2B;IACtC,UAAU,EAAE,qCAAqC;IACjD,YAAY,EAAE,iCAAiC;CACtC,CAAC;AAEX,MAAa,iBAAkB,SAAQ,qBAAa;IAGnD,YAAY,QAAkB,EAAE,GAAW,EAAE,aAAmB;QAC/D,KAAK,CAAC,QAAQ,EAAE,GAAG,EAAE,aAAa,CAAC,CAAC;QACpC,IAAI,CAAC,gBAAgB,GAAG,QAAQ,CAAC,aAAa,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;IAC7E,CAAC;IAED,UAAU;QACT,OAAO,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC;IAChC,CAAC;IAED,OAAO;QACN,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QACpC,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QAC1C,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAE7C,OAAO;YACN,OAAO,EAAE,WAAW;YACpB,WAAW;YACX,gBAAgB,EAAE;gBACjB,SAAS,EAAE,IAAI,CAAC,YAAY,EAAE;aAC9B;YACD,SAAS,EAAE;gBACV,KAAK;gBACL,MAAM;gBACN,IAAI,EAAE,aAAa;gBACnB,WAAW;aACX;SACD,CAAC;IACH,CAAC;IAEO,YAAY;QACnB,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAC7D,OAAO,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,oBAAoB,CAAC;IAC7D,CAAC;IAEO,aAAa;QACpB,MAAM,eAAe,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QACtE,IAAI,CAAC,eAAe;YAAE,OAAO,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAErD,MAAM,IAAI,GAAG,eAAe,CAAC,aAAa,CAAC,SAAS,CAAC,WAAW,CAAC,EAAE,YAAY,CAAC,SAAS,CAAC,CAAC;QAC3F,MAAM,MAAM,GAAG,eAAe,CAAC,aAAa,CAAC,SAAS,CAAC,aAAa,CAAC,EAAE,YAAY,CAAC,SAAS,CAAC,CAAC;QAE/F,IAAI,IAAI,IAAI,MAAM;YAAE,OAAO,GAAG,IAAI,MAAM,MAAM,GAAG,CAAC;QAClD,OAAO,IAAI,IAAI,MAAM,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAClD,CAAC;IAEO,gBAAgB;QACvB,mEAAmE;QACnE,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAC7E,OAAO,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC;IAC7D,CAAC;IAEO,oBAAoB;QAC3B,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,2BAA2B,CAAC,EAAE,YAAY,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;QACxG,uEAAuE;QACvE,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,oCAAoC,CAAC,CAAC;QAClE,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IAC5C,CAAC;IAEO,YAAY;QACnB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;QAC/C,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC9B,CAAC;IAEO,cAAc;QACrB,IAAI,CAAC,IAAI,CAAC,gBAAgB;YAAE,OAAO,EAAE,CAAC;QAEtC,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,IAAI,CAAgB,CAAC;QACnE,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;QAEzB,OAAO,8BAA8B,IAAA,mBAAa,EAAC,KAAK,CAAC,YAAY,CAAC;IACvE,CAAC;IAEO,YAAY,CAAC,SAAsB;QAC1C,MAAM,QAAQ,GAAG,SAAS,CAAC,aAAa,IAAI,IAAI,CAAC,QAAQ,CAAC;QAE1D,gEAAgE;QAChE,IAAI,CAAC,qBAAqB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAChD,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAC5C,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QACzC,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAC7C,IAAI,CAAC,mBAAmB,CAAC,SAAS,CAAC,CAAC;QACpC,kEAAkE;QAClE,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAC3C,IAAI,CAAC,sBAAsB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QACjD,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC;IACvC,CAAC;IAEO,qBAAqB,CAAC,SAAsB,EAAE,QAAkB;QACvE,SAAS,CAAC,gBAAgB,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;YACpE,MAAM,UAAU,GAAG,QAAQ,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC;YACxD,UAAU,CAAC,SAAS,GAAG,gBAAgB,CAAC;YAExC,sBAAsB;YACtB,MAAM,UAAU,GAAG,KAAK,CAAC,aAAa,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;YAC5D,MAAM,WAAW,GAAG,UAAU,EAAE,gBAAgB,CAAC,GAAG,CAAC,CAAC;YACtD,MAAM,QAAQ,GAAG,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YAC7D,MAAM,MAAM,GAAG,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YAE3D,qBAAqB;YACrB,MAAM,WAAW,GAAG,KAAK,CAAC,aAAa,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;YAC9D,MAAM,SAAS,GAAG,WAAW,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YAEzD,iCAAiC;YACjC,IAAI,QAAQ,IAAI,MAAM,EAAE,CAAC;gBACxB,MAAM,IAAI,GAAG,QAAQ,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;gBAC5C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,QAAQ,IAAI,MAAM,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC;gBAC/D,UAAU,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YAC9B,CAAC;YAED,IAAI,SAAS,EAAE,CAAC;gBACf,MAAM,CAAC,GAAG,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;gBACtC,CAAC,CAAC,WAAW,GAAG,SAAS,CAAC;gBAC1B,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;YAC3B,CAAC;YAED,KAAK,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;QAC/B,CAAC,CAAC,CAAC;IACJ,CAAC;IAEO,iBAAiB,CAAC,SAAsB,EAAE,QAAkB;QACnE,SAAS,CAAC,gBAAgB,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;YAChE,MAAM,GAAG,GAAG,KAAK,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;YACvC,MAAM,IAAI,GAAG,KAAK,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YACzC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI;gBAAE,OAAO;YAE1B,mEAAmE;YACnE,IAAI,QAAQ,GAAG,EAAE,CAAC;YAClB,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;YACzD,IAAI,SAAS,EAAE,CAAC;gBACf,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;YACzB,CAAC;iBAAM,CAAC;gBACP,wDAAwD;gBACxD,MAAM,QAAQ,GAAG,KAAK,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;gBAC7C,QAAQ,GAAG,QAAQ,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YAChD,CAAC;YAED,kCAAkC;YAClC,MAAM,MAAM,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;YAC7C,MAAM,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YAC/C,IAAI,QAAQ,EAAE,CAAC;gBACd,OAAO,CAAC,YAAY,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;gBAC5C,OAAO,CAAC,SAAS,GAAG,YAAY,QAAQ,EAAE,CAAC;YAC5C,CAAC;YACD,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC;YAC7C,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;YAE5B,qCAAqC;YACrC,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QAC3B,CAAC,CAAC,CAAC;IACJ,CAAC;IAEO,cAAc,CAAC,SAAsB,EAAE,QAAkB;QAChE,wEAAwE;QACxE,SAAS,CAAC,gBAAgB,CAAC,wBAAwB,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE;YACrE,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;YAC3C,MAAM,IAAI,GAAG,MAAM,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YAC9C,IAAI,CAAC,IAAI;gBAAE,OAAO;YAElB,MAAM,SAAS,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;YAChD,SAAS,CAAC,WAAW,GAAG,IAAI,CAAC;YAC7B,MAAM,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;QAC/B,CAAC,CAAC,CAAC;IACJ,CAAC;IAEO,kBAAkB,CAAC,SAAsB,EAAE,QAAkB;QACpE,wEAAwE;QACxE,SAAS,CAAC,gBAAgB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;YAC1D,+BAA+B;YAC/B,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAChC,IAAI,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAAE,OAAO;YAEnD,0EAA0E;YAC1E,IAAI,GAAG,GAAG,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;YACxC,MAAM,GAAG,GAAG,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,IAAI,OAAO,CAAC;YAE5E,wBAAwB;YACxB,IAAI,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC5B,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC;YAC/C,CAAC;iBAAM,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC9B,GAAG,GAAG,GAAG,GAAG,aAAa,CAAC;YAC3B,CAAC;iBAAM,CAAC;gBACP,GAAG,GAAG,GAAG,GAAG,aAAa,CAAC;YAC3B,CAAC;YAED,MAAM,QAAQ,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;YAC/C,QAAQ,CAAC,YAAY,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;YAClC,QAAQ,CAAC,YAAY,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;YAElC,kCAAkC;YAClC,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;QAC9B,CAAC,CAAC,CAAC;IACJ,CAAC;IAEO,mBAAmB,CAAC,SAAsB;QACjD,SAAS,CAAC,gBAAgB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;YAC1D,MAAM,GAAG,GAAG,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;YACpC,IAAI,CAAC,GAAG;gBAAE,OAAO;YAEjB,IAAI,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC5B,GAAG,CAAC,YAAY,CAAC,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC,CAAC;YAClE,CAAC;iBAAM,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC9B,GAAG,CAAC,YAAY,CAAC,KAAK,EAAE,GAAG,GAAG,aAAa,CAAC,CAAC;YAC9C,CAAC;iBAAM,CAAC;gBACP,GAAG,CAAC,YAAY,CAAC,KAAK,EAAE,GAAG,GAAG,aAAa,CAAC,CAAC;YAC9C,CAAC;QACF,CAAC,CAAC,CAAC;IACJ,CAAC;IAEO,sBAAsB,CAAC,SAAsB,EAAE,QAAkB;QACxE,0FAA0F;QAC1F,MAAM,SAAS,GAAG,CAAC,CAAC;QACpB,MAAM,YAAY,GAAG,CAAC,CAAC;QAEvB,SAAS,CAAC,gBAAgB,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;YACpE,MAAM,CAAC,GAAG,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;YAEtC,mEAAmE;YACnE,MAAM,WAAW,GAAG,CAAC,IAAU,EAAQ,EAAE;gBACxC,IAAI,IAAI,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;oBACjC,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,CAAC;gBAChE,CAAC;qBAAM,IAAI,IAAI,CAAC,QAAQ,KAAK,YAAY,EAAE,CAAC;oBAC3C,MAAM,EAAE,GAAG,IAAe,CAAC;oBAC3B,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;oBAErC,IAAI,GAAG,KAAK,QAAQ,EAAE,CAAC;wBACtB,MAAM,MAAM,GAAG,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;wBAChD,MAAM,CAAC,WAAW,GAAG,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC;wBAC1C,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;oBACvB,CAAC;yBAAM,IAAI,GAAG,KAAK,GAAG,EAAE,CAAC;wBACxB,MAAM,IAAI,GAAG,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;wBACzC,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;wBACzD,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC;wBACxC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;oBACrB,CAAC;yBAAM,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC;wBAC3B,MAAM,IAAI,GAAG,QAAQ,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;wBAC5C,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC;wBACxC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;oBACrB,CAAC;yBAAM,CAAC;wBACP,kDAAkD;wBAClD,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC;oBACpD,CAAC;gBACF,CAAC;YACF,CAAC,CAAC;YAEF,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC;YACpD,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;QACpB,CAAC,CAAC,CAAC;IACJ,CAAC;IAEO,gBAAgB,CAAC,SAAsB,EAAE,QAAkB;QAClE,SAAS,CAAC,gBAAgB,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE;YAC/D,MAAM,MAAM,GAAG,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;YAChD,MAAM,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC;YAC5C,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;IACJ,CAAC;IAEO,qBAAqB,CAAC,SAAsB;QACnD,SAAS,CAAC,gBAAgB,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE;YACnE,EAAE,CAAC,eAAe,CAAC,iBAAiB,CAAC,CAAC;QACvC,CAAC,CAAC,CAAC;IACJ,CAAC;IAEO,iBAAiB;QACxB,MAAM,IAAI,GAAG,IAAI,CAAC,gBAAgB,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAC9D,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC9D,CAAC;CACD;AA/QD,8CA+QC"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { BaseExtractor } from './_base';
|
|
2
|
+
import { ExtractorResult } from '../types/extractors';
|
|
3
|
+
export declare class XOembedExtractor extends BaseExtractor {
|
|
4
|
+
canExtract(): boolean;
|
|
5
|
+
extract(): ExtractorResult;
|
|
6
|
+
canExtractAsync(): boolean;
|
|
7
|
+
extractAsync(): Promise<ExtractorResult>;
|
|
8
|
+
private extractOembed;
|
|
9
|
+
private tryExtractFxTwitter;
|
|
10
|
+
private fetchFxTwitter;
|
|
11
|
+
private buildArticleResult;
|
|
12
|
+
private buildTweetResult;
|
|
13
|
+
private renderTweet;
|
|
14
|
+
private applyMarkers;
|
|
15
|
+
private applyFacets;
|
|
16
|
+
private renderArticle;
|
|
17
|
+
private renderBlock;
|
|
18
|
+
private renderAtomicBlock;
|
|
19
|
+
private renderInlineContent;
|
|
20
|
+
private escapeHtml;
|
|
21
|
+
}
|