mwoffliner 1.13.0 → 1.14.1

package/README.md

@@ -12,14 +12,19 @@ any recent MediaWiki.
 Read [CONTRIBUTING.md](./CONTRIBUTING.md) to know more about
 MWoffliner development.
 
+User Help is available in the for a a
+[FAQ](https://github.com/openzim/mwoffliner/wiki/Frequently-Asked-Questions).
+
 [![NPM](https://nodei.co/npm/mwoffliner.png)](https://www.npmjs.com/package/mwoffliner)
 
 [![npm](https://img.shields.io/npm/v/mwoffliner.svg)](https://www.npmjs.com/package/mwoffliner)
-[![Docker](https://ghcr-badge.deta.dev/openzim/mwoffliner/latest_tag?label=container)](https://ghcr.io/openzim/mwoffliner)
+[![node](https://img.shields.io/node/v/mwoffliner.svg)](https://www.npmjs.com/package/mwoffliner)
+[![Docker](https://ghcr-badge.egpl.dev/openzim/mwoffliner/latest_tag?label=container)](https://ghcr.io/openzim/mwoffliner)
 [![Build Status](https://github.com/openzim/mwoffliner/workflows/CI/badge.svg?query=branch%3Amain)](https://github.com/openzim/mwoffliner/actions/workflows/ci.yml?query=branch%3Amain)
 [![codecov](https://codecov.io/gh/openzim/mwoffliner/branch/main/graph/badge.svg)](https://codecov.io/gh/openzim/mwoffliner)
 [![CodeFactor](https://www.codefactor.io/repository/github/openzim/mwoffliner/badge)](https://www.codefactor.io/repository/github/openzim/mwoffliner)
 [![License](https://img.shields.io/npm/l/mwoffliner.svg)](LICENSE)
+[![Join Slack](https://img.shields.io/badge/Join%20us%20on%20Slack%20%23mwoffliner-2EB67D)](https://slack.kiwix.org)
 
 ## Features
 
@@ -78,9 +83,10 @@ this:
 ## API
 
 MWoffliner provides also an API and therefore can be used as a NodeJS
-library. Here a stub example:
+library. Here a stub example that could go in your index.mjs file:
 ```javascript
-const mwoffliner = require('mwoffliner');
+import * as mwoffliner from 'mwoffliner';
+
 const parameters = {
     mwUrl: "https://es.wikipedia.org",
     adminEmail: "foo@bar.net",

package/lib/Downloader.d.ts

@@ -1,11 +1,9 @@
 /// <reference types="node" resolution-mode="require"/>
 import type { BackoffStrategy } from 'backoff';
-import { AxiosRequestConfig } from 'axios';
+import { AxiosRequestConfig, AxiosResponse } from 'axios';
 import S3 from './S3.js';
 import { Dump } from './Dump.js';
-import MediaWiki from './MediaWiki.js';
 interface DownloaderOpts {
-    mw: MediaWiki;
     uaString: string;
     speed: number;
     reqTimeout: number;
@@ -13,6 +11,8 @@ interface DownloaderOpts {
     s3?: S3;
     webp: boolean;
     backoffOptions?: BackoffOptions;
+    mwWikiPath?: string;
+    insecure?: boolean;
 }
 interface BackoffOptions {
     strategy: BackoffStrategy;
@@ -20,66 +20,70 @@ interface BackoffOptions {
     retryIf: (error?: any) => boolean;
     backoffHandler: (number: number, delay: number, error?: any) => void;
 }
-export interface MWCapabilities {
-    apiAvailable: boolean;
-    veApiAvailable: boolean;
-    coordinatesAvailable: boolean;
-    desktopRestApiAvailable: boolean;
-    mobileRestApiAvailable: boolean;
-}
-export declare const defaultStreamRequestOptions: AxiosRequestConfig;
+/**
+ * Downloader is a class providing content retrieval functionalities for both Mediawiki and S3 remote instances.
+ */
 declare class Downloader {
-    readonly mw: MediaWiki;
     loginCookie: string;
     readonly speed: number;
-    baseUrl: string;
-    baseUrlForMainPage: string;
     cssDependenceUrls: KVS<boolean>;
     readonly webp: boolean;
     readonly requestTimeout: number;
+    readonly basicRequestOptions: AxiosRequestConfig;
+    readonly arrayBufferRequestOptions: AxiosRequestConfig;
+    readonly jsonRequestOptions: AxiosRequestConfig;
+    readonly streamRequestOptions: AxiosRequestConfig;
+    wikimediaMobileJsDependenciesList: string[];
+    wikimediaMobileStyleDependenciesList: string[];
     private readonly uaString;
     private activeRequests;
     private maxActiveRequests;
-    private readonly urlPartCache;
     private readonly backoffOptions;
     private readonly optimisationCacheUrl;
     private s3;
-    private mwCapabilities;
-    arrayBufferRequestOptions: AxiosRequestConfig;
-    jsonRequestOptions: AxiosRequestConfig;
-    streamRequestOptions: AxiosRequestConfig;
-    constructor({ mw, uaString, speed, reqTimeout, optimisationCacheUrl, s3, webp, backoffOptions }: DownloaderOpts);
-    serializeUrl(url: string): string;
-    deserializeUrl(url: string): string;
-    setBaseUrls(): Promise<void>;
-    checkApiAvailabilty(url: string): Promise<boolean>;
-    checkCapabilities(testArticleId?: string): Promise<void>;
+    private apiUrlDirector;
+    private articleUrlDirector;
+    private mainPageUrlDirector;
+    private readonly insecure;
+    constructor({ uaString, speed, reqTimeout, optimisationCacheUrl, s3, webp, backoffOptions, insecure }: DownloaderOpts);
+    private getUrlDirector;
+    setUrlsDirectors(mainPageRenderer: any, articlesRenderer: any): void;
+    getArticleUrl(articleId: string): string;
+    getMainPageUrl(articleId: string): string;
     removeEtagWeakPrefix(etag: string): string;
-    query(query: string): KVS<any>;
+    query(): KVS<any>;
     getArticleDetailsIds(articleIds: string[], shouldGetThumbnail?: boolean): Promise<QueryMwRet>;
     getArticleDetailsNS(ns: number, gapcontinue?: string): Promise<{
         gapContinue: string;
         articleDetails: QueryMwRet;
     }>;
-    getArticle(articleId: string, dump: Dump, articleDetailXId: RKVS<ArticleDetail>, articleDetail?: ArticleDetail): Promise<RenderedArticle[]>;
+    getArticle(webp: boolean, _moduleDependencies: any, articleId: string, articleDetailXId: RKVS<ArticleDetail>, articleRenderer: any, articleUrl: any, dump: Dump, articleDetail?: ArticleDetail, isMainPage?: boolean): Promise<any>;
     getJSON<T>(_url: string): Promise<T>;
-    downloadContent(_url: string, retry?: boolean): Promise<{
+    request<T = any, R extends AxiosResponse<T> = AxiosResponse<T>, D = any>(config: AxiosRequestConfig<D>): Promise<R>;
+    downloadContent(_url: string, kind: DonwloadKind, retry?: boolean): Promise<{
         content: Buffer | string;
-        responseHeaders: any;
+        contentType: string;
+        setCookie: string | null;
     }>;
     canGetUrl(url: string): Promise<boolean>;
-    private getArticleUrl;
     private static handleMWWarningsAndErrors;
     private getArticleQueryOpts;
     private setArticleSubCategories;
     private claimRequest;
     private releaseRequest;
     private getJSONCb;
+    private getImageMimeType;
     private getCompressedBody;
     private getContentCb;
     private downloadImage;
     private errHandler;
     private getSubCategories;
     private backoffCall;
+    getModuleDependencies(title: string): Promise<{
+        jsConfigVars: string;
+        jsDependenciesList: string[];
+        styleDependenciesList: string[];
+    }>;
+    private streamToBuffer;
 }
 export default Downloader;