mwoffliner 1.12.1 → 1.14.0

package/README.md

@@ -1,25 +1,30 @@
 # MWoffliner
 
 MWoffliner is a tool for making a local offline HTML snapshot of any
-online [Mediawiki](https://mediawiki.org) instance. It goes through
+online [MediaWiki](https://mediawiki.org) instance. It goes through
 all online articles (or a selection if specified) and create the
 corresponding [ZIM](https://openzim.org) file. It has mainly been
 tested against Wikimedia projects like
-[Wikipedia](https://wikipedia.org),
-[Wiktionary](https://wiktionary.org), ... But it should also work for
-any recent Mediawiki.
+[Wikipedia](https://wikipedia.org) and
+[Wiktionary](https://wiktionary.org) --- but it should also work for
+any recent MediaWiki.
 
 Read [CONTRIBUTING.md](./CONTRIBUTING.md) to know more about
 MWoffliner development.
 
+User Help is available in the for a a
+[FAQ](https://github.com/openzim/mwoffliner/wiki/Frequently-Asked-Questions).
+
 [![NPM](https://nodei.co/npm/mwoffliner.png)](https://www.npmjs.com/package/mwoffliner)
 
 [![npm](https://img.shields.io/npm/v/mwoffliner.svg)](https://www.npmjs.com/package/mwoffliner)
-[![Docker Image](https://img.shields.io/docker/v/openzim/mwoffliner?arch=amd64&label=docker&sort=semver)](https://hub.docker.com/r/openzim/mwoffliner)
+[![node](https://img.shields.io/node/v/mwoffliner.svg)](https://www.npmjs.com/package/mwoffliner)
+[![Docker](https://ghcr-badge.egpl.dev/openzim/mwoffliner/latest_tag?label=container)](https://ghcr.io/openzim/mwoffliner)
 [![Build Status](https://github.com/openzim/mwoffliner/workflows/CI/badge.svg?query=branch%3Amain)](https://github.com/openzim/mwoffliner/actions/workflows/ci.yml?query=branch%3Amain)
 [![codecov](https://codecov.io/gh/openzim/mwoffliner/branch/main/graph/badge.svg)](https://codecov.io/gh/openzim/mwoffliner)
 [![CodeFactor](https://www.codefactor.io/repository/github/openzim/mwoffliner/badge)](https://www.codefactor.io/repository/github/openzim/mwoffliner)
 [![License](https://img.shields.io/npm/l/mwoffliner.svg)](LICENSE)
+[![Join Slack](https://img.shields.io/badge/Join%20us%20on%20Slack%20%23mwoffliner-2EB67D)](https://slack.kiwix.org)
 
 ## Features
 
@@ -42,7 +47,7 @@ Run `mwoffliner --help` to get all the possible options.
   machine (packages `libjpeg-dev`, `libglu1`, `autoconf`, `automake`, `gcc` on
   Debian/Ubuntu)
 
-... and an online Mediawiki with its API available.
+... and an online MediaWiki with its API available.
 
 ## Usage
 
@@ -55,7 +60,7 @@ You might need to run this command with the `sudo` command, depending
 how your `npm` is configured.
 
 `npm` permission checking can be a bit annoying for a
-newcommer. Please read the documentation carefully if you hit
+newcomer. Please read the documentation carefully if you hit
 problems: https://docs.npmjs.com/cli/v7/using-npm/scripts#user
 
 Then to run it:
@@ -63,6 +68,12 @@ Then to run it:
 mwoffliner --help
 ```
 
+To install and run it locally:
+```bash
+npm i
+npm run mwoffliner -- --help
+```
+
 To use MWoffliner with a S3 cache, you should provide a S3 URL like
 this:
 ```bash
@@ -72,9 +83,10 @@ this:
 ## API
 
 MWoffliner provides also an API and therefore can be used as a NodeJS
-library. Here a stub example:
+library. Here a stub example that could go in your index.mjs file:
 ```javascript
-const mwoffliner = require('mwoffliner');
+import * as mwoffliner from 'mwoffliner';
+
 const parameters = {
     mwUrl: "https://es.wikipedia.org",
     adminEmail: "foo@bar.net",

package/lib/Downloader.d.ts

@@ -3,9 +3,7 @@ import type { BackoffStrategy } from 'backoff';
 import { AxiosRequestConfig } from 'axios';
 import S3 from './S3.js';
 import { Dump } from './Dump.js';
-import MediaWiki from './MediaWiki.js';
 interface DownloaderOpts {
-    mw: MediaWiki;
     uaString: string;
     speed: number;
     reqTimeout: number;
@@ -13,6 +11,8 @@ interface DownloaderOpts {
     s3?: S3;
     webp: boolean;
     backoffOptions?: BackoffOptions;
+    mwWikiPath?: string;
+    insecure?: boolean;
 }
 interface BackoffOptions {
     strategy: BackoffStrategy;
@@ -20,55 +20,50 @@ interface BackoffOptions {
     retryIf: (error?: any) => boolean;
     backoffHandler: (number: number, delay: number, error?: any) => void;
 }
-export interface MWCapabilities {
-    apiAvailable: boolean;
-    veApiAvailable: boolean;
-    coordinatesAvailable: boolean;
-    desktopRestApiAvailable: boolean;
-    mobileRestApiAvailable: boolean;
-}
+export declare const defaultStreamRequestOptions: AxiosRequestConfig;
+/**
+ * Downloader is a class providing content retrieval functionalities for both Mediawiki and S3 remote instances.
+ */
 declare class Downloader {
-    readonly mw: MediaWiki;
     loginCookie: string;
     readonly speed: number;
-    baseUrl: string;
-    baseUrlForMainPage: string;
     cssDependenceUrls: KVS<boolean>;
     readonly webp: boolean;
     readonly requestTimeout: number;
+    arrayBufferRequestOptions: AxiosRequestConfig;
+    jsonRequestOptions: AxiosRequestConfig;
+    streamRequestOptions: AxiosRequestConfig;
+    wikimediaMobileJsDependenciesList: string[];
+    wikimediaMobileStyleDependenciesList: string[];
     private readonly uaString;
     private activeRequests;
     private maxActiveRequests;
-    private readonly urlPartCache;
     private readonly backoffOptions;
     private readonly optimisationCacheUrl;
     private s3;
-    private mwCapabilities;
-    arrayBufferRequestOptions: AxiosRequestConfig;
-    jsonRequestOptions: AxiosRequestConfig;
-    streamRequestOptions: AxiosRequestConfig;
-    constructor({ mw, uaString, speed, reqTimeout, optimisationCacheUrl, s3, webp, backoffOptions }: DownloaderOpts);
-    serializeUrl(url: string): string;
-    deserializeUrl(url: string): string;
-    setBaseUrls(): Promise<void>;
-    checkApiAvailabilty(url: string): Promise<boolean>;
-    checkCapabilities(testArticleId?: string): Promise<void>;
+    private apiUrlDirector;
+    private articleUrlDirector;
+    private mainPageUrlDirector;
+    private readonly insecure;
+    constructor({ uaString, speed, reqTimeout, optimisationCacheUrl, s3, webp, backoffOptions, insecure }: DownloaderOpts);
+    private getUrlDirector;
+    setUrlsDirectors(mainPageRenderer: any, articlesRenderer: any): void;
+    getArticleUrl(articleId: string): string;
+    getMainPageUrl(articleId: string): string;
     removeEtagWeakPrefix(etag: string): string;
-    query(query: string): KVS<any>;
+    query(): KVS<any>;
     getArticleDetailsIds(articleIds: string[], shouldGetThumbnail?: boolean): Promise<QueryMwRet>;
     getArticleDetailsNS(ns: number, gapcontinue?: string): Promise<{
         gapContinue: string;
         articleDetails: QueryMwRet;
     }>;
-    getArticle(articleId: string, dump: Dump, articleDetailXId: RKVS<ArticleDetail>, articleDetail?: ArticleDetail): Promise<RenderedArticle[]>;
+    getArticle(webp: boolean, _moduleDependencies: any, articleId: string, articleDetailXId: RKVS<ArticleDetail>, articleRenderer: any, articleUrl: any, dump: Dump, articleDetail?: ArticleDetail, isMainPage?: boolean): Promise<any>;
     getJSON<T>(_url: string): Promise<T>;
     downloadContent(_url: string, retry?: boolean): Promise<{
         content: Buffer | string;
         responseHeaders: any;
     }>;
     canGetUrl(url: string): Promise<boolean>;
-    private getArticleUrl;
-    private stripNonContinuedProps;
     private static handleMWWarningsAndErrors;
     private getArticleQueryOpts;
     private setArticleSubCategories;
@@ -81,5 +76,11 @@ declare class Downloader {
     private errHandler;
     private getSubCategories;
     private backoffCall;
+    getModuleDependencies(title: string): Promise<{
+        jsConfigVars: string;
+        jsDependenciesList: string[];
+        styleDependenciesList: string[];
+    }>;
+    private streamToBuffer;
 }
 export default Downloader;