mwoffliner 1.11.11 → 1.12.0

package/README.md

@@ -16,8 +16,8 @@ MWoffliner development.
 
 [![npm](https://img.shields.io/npm/v/mwoffliner.svg)](https://www.npmjs.com/package/mwoffliner)
 [![Docker Image](https://img.shields.io/docker/v/openzim/mwoffliner?arch=amd64&label=docker&sort=semver)](https://hub.docker.com/r/openzim/mwoffliner)
-[![Build Status](https://github.com/openzim/mwoffliner/workflows/CI/badge.svg?query=branch%3Amaster)](https://github.com/openzim/mwoffliner/actions/workflows/ci.yml?query=branch%3Amaster)
-[![codecov](https://codecov.io/gh/openzim/mwoffliner/branch/master/graph/badge.svg)](https://codecov.io/gh/openzim/mwoffliner)
+[![Build Status](https://github.com/openzim/mwoffliner/workflows/CI/badge.svg?query=branch%3Amain)](https://github.com/openzim/mwoffliner/actions/workflows/ci.yml?query=branch%3Amain)
+[![codecov](https://codecov.io/gh/openzim/mwoffliner/branch/main/graph/badge.svg)](https://codecov.io/gh/openzim/mwoffliner)
 [![CodeFactor](https://www.codefactor.io/repository/github/openzim/mwoffliner/badge)](https://www.codefactor.io/repository/github/openzim/mwoffliner)
 [![License](https://img.shields.io/npm/l/mwoffliner.svg)](LICENSE)
 
@@ -36,10 +36,10 @@ Run `mwoffliner --help` to get all the possible options.
 
 - *NIX Operating System (GNU/Linux, macOS, ...)
 - [Redis](https://redis.io/)
-- [NodeJS](https://nodejs.org/en/) version 12 or greater
+- [NodeJS](https://nodejs.org/en/) version 16 or greater
 - [Libzim](https://github.com/openzim/libzim) (On GNU/Linux & macOS we automatically download it)
 - Various build tools which are probably already installed on your
-  machine (packages `libjpeg-dev`, `autoconf`, `automake`, `gcc` on
+  machine (packages `libjpeg-dev`, `libglu1`, `autoconf`, `automake`, `gcc` on
   Debian/Ubuntu)
 
 ... and an online Mediawiki with its API available.
@@ -95,22 +95,11 @@ Complementary information about MWoffliner:
 * Wikitext is the name of the markup language that MediaWiki uses.
 * MediaWiki includes a parser for WikiText into HTML, and this
   parser creates the HTML pages displayed in your browser.
-* There is another WikiText parser, called Parsoid, implemented in
-  Javascript/NodeJS. MWoffliner uses Parsoid.
-* Parsoid is planned to eventually become the main parser for
-  MediaWiki.
-* MWoffliner calls Parsoid and then post-processes the results for
-  offline format.
 
 ### GNU/Linux - Debian based distributions
 
 Install NodeJS:
-```bash
-curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.33.11/install.sh | bash && \
-source ~/.bashrc && \
-nvm install stable && \
-node --version
-```
+Read https://nodejs.org/en/download/current/
 
 Install Redis:
 ```bash

package/jest.config.cjs

@@ -0,0 +1,18 @@
+module.exports = {
+  extensionsToTreatAsEsm: ['.ts'],
+  moduleNameMapper: {
+    '^(\\.{1,2}/.*)\\.js$': '$1',
+  },
+  moduleFileExtensions: ["ts", "js"],
+  collectCoverage: false,
+  verbose: true,
+  transform: {
+    '^.+\\.ts?$': [
+      'ts-jest',
+      {
+        useESM: true,
+      },
+    ],
+  },
+}
+  

package/lib/DOMUtils.js

@@ -1,5 +1,3 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
 const DOMUtils = {
     deleteNode(node) {
         if (node.parentNode) {
@@ -23,5 +21,5 @@ const DOMUtils = {
         return sibling;
     },
 };
-exports.default = DOMUtils;
+export default DOMUtils;
 //# sourceMappingURL=DOMUtils.js.map

package/lib/DOMUtils.js.map

@@ -1 +1 @@
-{"version":3,"file":"DOMUtils.js","sourceRoot":"","sources":["../src/DOMUtils.ts"],"names":[],"mappings":";;AAAA,MAAM,QAAQ,GAAG;IACf,UAAU,CAAC,IAAmB;QAC5B,IAAI,IAAI,CAAC,UAAU,EAAE;YACnB,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;SACnC;aAAM;YACL,IAAI,CAAC,SAAS,GAAG,EAAE,CAAC;SACrB;QACD,IAAI,GAAG,SAAS,CAAC;IACnB,CAAC;IAED,YAAY,CAAC,IAAmB,EAAE,IAAY,EAAE,GAAQ;QACtD,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QACvC,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;QACnD,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,QAAe,CAAC,CAAC;IAC3C,CAAC;IAED,kBAAkB,CAAC,IAAmB;QACpC,IAAI,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC;QAC/B,OAAO,OAAO,IAAI,OAAO,CAAC,QAAQ,KAAK,CAAC,CAAC,kBAAkB,EAAE;YAC3D,OAAO,GAAG,OAAO,CAAC,WAAW,CAAC;SAC/B;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;CACF,CAAC;AAEF,kBAAe,QAAQ,CAAC"}
+{"version":3,"file":"DOMUtils.js","sourceRoot":"","sources":["../src/DOMUtils.ts"],"names":[],"mappings":"AAAA,MAAM,QAAQ,GAAG;IACf,UAAU,CAAC,IAAmB;QAC5B,IAAI,IAAI,CAAC,UAAU,EAAE;YACnB,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,IAAI,CAAC,CAAA;SAClC;aAAM;YACL,IAAI,CAAC,SAAS,GAAG,EAAE,CAAA;SACpB;QACD,IAAI,GAAG,SAAS,CAAA;IAClB,CAAC;IAED,YAAY,CAAC,IAAmB,EAAE,IAAY,EAAE,GAAQ;QACtD,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAA;QACtC,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAA;QAClD,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,QAAe,CAAC,CAAA;IAC1C,CAAC;IAED,kBAAkB,CAAC,IAAmB;QACpC,IAAI,OAAO,GAAG,IAAI,CAAC,WAAW,CAAA;QAC9B,OAAO,OAAO,IAAI,OAAO,CAAC,QAAQ,KAAK,CAAC,CAAC,kBAAkB,EAAE;YAC3D,OAAO,GAAG,OAAO,CAAC,WAAW,CAAA;SAC9B;QACD,OAAO,OAAO,CAAA;IAChB,CAAC;CACF,CAAA;AAED,eAAe,QAAQ,CAAA"}

package/lib/Downloader.d.ts

@@ -1,16 +1,14 @@
-/// <reference types="node" />
+/// <reference types="node" resolution-mode="require"/>
 import type { BackoffStrategy } from 'backoff';
 import { AxiosRequestConfig } from 'axios';
-import S3 from './S3';
-import { Dump } from './Dump';
-import MediaWiki from './MediaWiki';
+import S3 from './S3.js';
+import { Dump } from './Dump.js';
+import MediaWiki from './MediaWiki.js';
 interface DownloaderOpts {
     mw: MediaWiki;
     uaString: string;
     speed: number;
     reqTimeout: number;
-    noLocalParserFallback: boolean;
-    forceLocalParser: boolean;
     optimisationCacheUrl: string;
     s3?: S3;
     webp: boolean;
@@ -37,12 +35,10 @@ declare class Downloader {
     baseUrlForMainPage: string;
     cssDependenceUrls: KVS<boolean>;
     readonly webp: boolean;
+    readonly requestTimeout: number;
     private readonly uaString;
     private activeRequests;
     private maxActiveRequests;
-    private readonly requestTimeout;
-    private readonly noLocalParserFallback;
-    private readonly forceLocalParser;
     private readonly urlPartCache;
     private readonly backoffOptions;
     private readonly optimisationCacheUrl;
@@ -51,23 +47,22 @@ declare class Downloader {
     arrayBufferRequestOptions: AxiosRequestConfig;
     jsonRequestOptions: AxiosRequestConfig;
     streamRequestOptions: AxiosRequestConfig;
-    constructor({ mw, uaString, speed, reqTimeout, noLocalParserFallback, forceLocalParser: forceLocalParser, optimisationCacheUrl, s3, webp, backoffOptions }: DownloaderOpts);
+    constructor({ mw, uaString, speed, reqTimeout, optimisationCacheUrl, s3, webp, backoffOptions }: DownloaderOpts);
     serializeUrl(url: string): string;
     deserializeUrl(url: string): string;
     setBaseUrls(): Promise<void>;
     checkApiAvailabilty(url: string): Promise<boolean>;
     checkCapabilities(testArticleId?: string): Promise<void>;
     removeEtagWeakPrefix(etag: string): string;
-    initLocalServices(): Promise<void>;
     query(query: string): KVS<any>;
     getArticleDetailsIds(articleIds: string[], shouldGetThumbnail?: boolean): Promise<QueryMwRet>;
     getArticleDetailsNS(ns: number, gapcontinue?: string): Promise<{
         gapContinue: string;
         articleDetails: QueryMwRet;
     }>;
-    getArticle(articleId: string, dump: Dump): Promise<RenderedArticle[]>;
+    getArticle(articleId: string, dump: Dump, articleDetailXId: RKVS<ArticleDetail>, articleDetail?: ArticleDetail): Promise<RenderedArticle[]>;
     getJSON<T>(_url: string): Promise<T>;
-    downloadContent(_url: string): Promise<{
+    downloadContent(_url: string, retry?: boolean): Promise<{
         content: Buffer | string;
         responseHeaders: any;
     }>;