mindee 4.0.1 → 4.1.0

package/CHANGELOG.md

@@ -1,5 +1,30 @@
 # CHANGELOG
 
+## v4.1.0 - 2023-08-31
+### Changes
+* :boom: remove support for node versions <16
+* :sparkles: add support for auto-polling in asynchronous calls
+* :sparkles: update HTTP error management system
+* :sparkles: add possibility to access raw server response
+* :arrow_up: add support for node 20
+* :wrench: expose more internal tools
+* :memo: add better technical documentation
+* :recycle: remove deprecated examples
+* :recycle: update linting rules
+* :arrow_up: update testing library
+
+### Fixes
+* :bug: fix miscellaneous bugs related to http parsing issues
+
+
+## v4.0.2 - 2023-08-24
+### Changes
+* :recycle: updated technical documentation
+
+### Fixes
+* :bug: fix url source document not being sent properly
+
+
 ## v4.0.1 - 2023-08-22
 ### Fixes
 * :bug: fixed pre-commit dependency script blocking installs

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "mindee",
-    "version": "4.0.1",
+    "version": "4.1.0",
     "description": "Mindee Client Library for Node.js",
     "main": "src/index.js",
     "bin": "bin/mindee.js",
@@ -23,7 +23,7 @@
         "CHANGELOG.md"
     ],
     "engines": {
-        "node": ">= 14"
+        "node": ">= 16"
     },
     "repository": {
         "type": "git",
@@ -46,9 +46,10 @@
         "@typescript-eslint/eslint-plugin": "^5.57.1",
         "@typescript-eslint/parser": "^5.57.1",
         "chai": "^4.3.7",
-        "eslint": "^8.32.0",
+        "eslint": "^8.47.0",
         "lint-staged": "^13.2.3",
         "mocha": "^10.1.0",
+        "nock": "^13.3.3",
         "ts-node": "^10.9.1",
         "typedoc": "~0.23",
         "typescript": "^5.0.4"

package/src/cli.js

@@ -193,8 +193,7 @@ function getPredictParams(options) {
     };
     return predictParams;
 }
-async function callParse(command, inputPath, options) {
-    const conf = getConfig(command);
+async function callParse(productClass, command, inputPath, options) {
     const mindeeClient = initClient(options);
     const predictParams = getPredictParams(options);
     const pageOptions = getPageOptions(options);
@@ -202,7 +201,7 @@ async function callParse(command, inputPath, options) {
     let response;
     if (command === COMMAND_CUSTOM) {
         const customEndpoint = mindeeClient.createEndpoint(options.endpoint, options.account, options.version);
-        response = await mindeeClient.parse(conf.docClass, inputSource, {
+        response = await mindeeClient.parse(productClass, inputSource, {
             endpoint: customEndpoint,
             pageOptions: pageOptions,
             allWords: predictParams.allWords,
@@ -210,7 +209,7 @@ async function callParse(command, inputPath, options) {
         });
     }
     else {
-        response = await mindeeClient.parse(conf.docClass, inputSource, {
+        response = await mindeeClient.parse(productClass, inputSource, {
             pageOptions: pageOptions,
             allWords: predictParams.allWords,
             cropper: predictParams.cropper,
@@ -218,8 +217,7 @@ async function callParse(command, inputPath, options) {
     }
     printResponse(response.document, options);
 }
-async function callEnqueue(command, inputPath, options) {
-    const conf = getConfig(command);
+async function callEnqueueAndParse(productClass, command, inputPath, options) {
     const mindeeClient = initClient(options);
     const predictParams = getPredictParams(options);
     const pageOptions = getPageOptions(options);
@@ -227,32 +225,30 @@ async function callEnqueue(command, inputPath, options) {
     let response;
     if (command === COMMAND_CUSTOM) {
         const customEndpoint = mindeeClient.createEndpoint(options.endpoint, options.account, options.version);
-        response = await mindeeClient.enqueue(conf.docClass, inputSource, {
+        response = await mindeeClient.enqueueAndParse(productClass, inputSource, {
             endpoint: customEndpoint,
             pageOptions: pageOptions,
             allWords: predictParams.allWords,
             cropper: predictParams.cropper,
+            initialDelaySec: 6,
+            delaySec: 3,
+            maxRetries: 10,
         });
     }
     else {
-        response = await mindeeClient.enqueue(conf.docClass, inputSource, {
+        response = await mindeeClient.enqueueAndParse(productClass, inputSource, {
             pageOptions: pageOptions,
             allWords: predictParams.allWords,
             cropper: predictParams.cropper,
+            initialDelaySec: 6,
+            delaySec: 3,
+            maxRetries: 10,
         });
-    }
-    console.log(response.job);
-}
-async function callParseQueued(command, queueId, options) {
-    const conf = getConfig(command);
-    const mindeeClient = initClient(options);
-    const response = await mindeeClient.parseQueued(conf.docClass, queueId);
-    if (response.document !== undefined) {
+        if (!response.document) {
+            throw Error("Document could not be retrieved");
+        }
         printResponse(response.document, options);
     }
-    else {
-        console.log(response.job);
-    }
 }
 function printResponse(document, options) {
     if (options.allWords) {
@@ -291,20 +287,11 @@ function addDisplayOptions(prog) {
     prog.option("-p, --pages", "show content of individual pages");
 }
 function routeSwitchboard(command, inputPath, allOptions) {
-    switch (command.parent?.name()) {
-        case "parse": {
-            return callParse(command.name(), inputPath, allOptions);
-        }
-        case "enqueue": {
-            return callEnqueue(command.name(), inputPath, allOptions);
-        }
-        case "parse-queued": {
-            return callParseQueued(command.name(), inputPath, allOptions);
-        }
-        default: {
-            throw new Error("Unhandled parent command.");
-        }
+    const docClass = getConfig(command.name()).docClass;
+    if ("async" in command.opts() && command.opts()["async"]) {
+        return callEnqueueAndParse(docClass, command.name(), inputPath, allOptions);
     }
+    return callParse(docClass, command.name(), inputPath, allOptions);
 }
 function addAction(prog) {
     if (prog.name() === COMMAND_CUSTOM) {
@@ -333,44 +320,27 @@ function addAction(prog) {
 function cli() {
     program.name("mindee");
     program.option("-d, --debug", "high verbosity mode");
-    const predict = program.command("parse").description("Parse synchronously.");
-    addMainOptions(predict);
-    const enqueue = program
-        .command("enqueue")
-        .description("Add to async parse queue.");
-    addMainOptions(enqueue);
-    const parseQueued = program
-        .command("parse-queued")
-        .description("Parse from async queue.");
-    addMainOptions(parseQueued);
     CLI_COMMAND_CONFIG.forEach((info, name) => {
-        if (info.sync) {
-            const prog = predict
-                .command(name)
-                .description(`Parse an ${info.displayName}.`);
-            if (name === COMMAND_CUSTOM) {
-                addCustomPostOptions(prog);
-            }
-            addDisplayOptions(prog);
-            addPostOptions(prog, info);
-            addAction(prog);
-        }
+        const prog = program.command(name);
+        prog.description(`${info.displayName} document`);
         if (info.async) {
-            const progEnqueue = enqueue
-                .command(name)
-                .description(`Add an ${info.displayName} to the queue.`);
-            if (name === COMMAND_CUSTOM) {
-                addCustomPostOptions(progEnqueue);
+            const asyncOpt = new commander_1.Option("-A, --async", "Call asynchronously");
+            if (info.sync) {
+                asyncOpt.default(false);
+            }
+            else {
+                asyncOpt.default(true);
+                asyncOpt.hideHelp();
             }
-            addPostOptions(progEnqueue, info);
-            addAction(progEnqueue);
-            const progParse = parseQueued
-                .command(name)
-                .description(`Parse an ${info.displayName} from the queue.`)
-                .argument("<doc_id>", "ID of the document");
-            addDisplayOptions(progParse);
-            addAction(progParse);
+            prog.addOption(asyncOpt);
+        }
+        if (name === COMMAND_CUSTOM) {
+            addCustomPostOptions(prog);
         }
+        addMainOptions(prog);
+        addDisplayOptions(prog);
+        addPostOptions(prog, info);
+        addAction(prog);
     });
     program.parse(process.argv);
 }

package/src/client.d.ts

@@ -4,10 +4,11 @@ import { Readable } from "stream";
 import { InputSource, PageOptions } from "./input";
 import { Endpoint } from "./http";
 import { Inference, AsyncPredictResponse, StringDict, PredictResponse } from "./parsing/common";
+/**
+ * Options relating to predictions.
+ */
 export interface PredictOptions {
-    endpointName?: string;
-    accountName?: string;
-    endpointVersion?: string;
+    /** A custom endpoint. */
     endpoint?: Endpoint;
     /**
      * Whether to include the full text for each page.
@@ -21,18 +22,19 @@ export interface PredictOptions {
      * This performs a cropping operation on the server and will increase response time.
      */
     cropper?: boolean;
-    pageOptions?: PageOptions;
-}
-export interface CustomConfigParams {
-    /** Your organization's username on the API Builder. */
-    accountName: string;
-    /** The "API name" field in the "Settings" page of the API Builder. */
-    endpointName: string;
     /**
-     * If set, locks the version of the model to use.
-     * If not set, use the latest version of the model.
+     * If set, remove pages from the document as specified.
+     * This is done before sending the file to the server and is useful to avoid page limitations.
      */
-    version?: string;
+    pageOptions?: PageOptions;
+}
+/**
+ * Asynchronous polling parameters.
+ */
+export interface AsyncOptions extends PredictOptions {
+    initialDelaySec: number;
+    delaySec: number;
+    maxRetries: number;
 }
 export interface ClientOptions {
     /** Your API key for all endpoints. */
@@ -43,35 +45,77 @@ export interface ClientOptions {
     debug?: boolean;
 }
 /**
- * Mindee Client
+ * Mindee Client class that centralizes most basic operations.
+ *
+ * @category Client
  */
 export declare class Client {
     #private;
+    /** Key of the API. */
     protected apiKey: string;
     /**
-     * @param options
+     * @param options options for the initialization of a client.
      */
     constructor({ apiKey, throwOnError, debug }?: ClientOptions);
     /**
      * Send a document to a synchronous endpoint and parse the predictions.
-     * @param productClass
-     * @param params
+     *
+     * @param productClass product class to use for calling the API and parsing the response.
+     * @param inputSource document to parse.
+     * @param params parameters relating to prediction options.
+     * @typeParam T an extension of an `Inference`. Can be omitted as it will be inferred from the `productClass`.
+     * @category Synchronous
+     * @returns a `Promise` containing parsing results.
      */
     parse<T extends Inference>(productClass: new (httpResponse: StringDict) => T, inputSource: InputSource, params?: PredictOptions): Promise<PredictResponse<T>>;
     /**
      * Send the document to an asynchronous endpoint and return its ID in the queue.
-     * @param productClass
-     * @param params
+     * @param productClass product class to use for calling  the API and parsing the response.
+     * @param params parameters relating to prediction options.
+     * @category Asynchronous
+     * @returns a `Promise` containing the job (queue) corresponding to a document.
      */
     enqueue<T extends Inference>(productClass: new (httpResponse: StringDict) => T, inputSource: InputSource, params?: PredictOptions): Promise<AsyncPredictResponse<T>>;
+    /**
+     * Polls a queue and returns its status as well as the prediction results if the parsing is done.
+     *
+     * @param productClass product class to use for calling  the API and parsing the response.
+     * @param queueId id of the queue to poll.
+     * @param params parameters relating to prediction options.
+     * @typeParam T an extension of an `Inference`. Can be omitted as it will be inferred from the `productClass`.
+     * @category Asynchronous
+     * @returns a `Promise` containing a `Job`, which also contains a `Document` if the
+     * parsing is complete.
+     */
     parseQueued<T extends Inference>(productClass: new (httpResponse: StringDict) => T, queueId: string, params?: PredictOptions): Promise<AsyncPredictResponse<T>>;
+    /**
+     * Send a document to an asynchronous endpoint and poll the server until the result is sent or
+     * until the maximum amount of tries is reached.
+     *
+     * @param productClass product class to use for calling the API and parsing the response.
+     * @param inputSource document to parse.
+     * @param asyncParams parameters relating to prediction options.
+     *
+     * @typeParam T an extension of an `Inference`. Can be omitted as it will be inferred from the `productClass`.
+     * @category Synchronous
+     * @returns a `Promise` containing parsing results.
+     */
+    enqueueAndParse<T extends Inference>(productClass: new (httpResponse: StringDict) => T, inputSource: InputSource, asyncParams?: AsyncOptions): Promise<AsyncPredictResponse<T>>;
+    /**
+     * Forces boolean coercion on truthy/falsy parameters.
+     * @param param input parameter to check.
+     * @returns a strict boolean value.
+     */
     protected getBooleanParam(param?: boolean): boolean;
     /**
      * Creates a custom endpoint with the given values. Raises an error if the endpoint is invalid.
-     * @param productClass Class of the product
-     * @param endpointName Name of a custom Endpoint
-     * @param accountName Name of the account tied to the active Endpoint
-     * @param version Version of a custom Endpoint
+     * @param productClass product class to use for calling the API and parsing the response.
+     * @param endpointName Name of the custom Endpoint.
+     * @param accountName Name of the account tied to the Endpoint.
+     * @param version Version of the custom Endpoint.
+     * @typeParam T an extension of an `Inference`. Can be omitted as it will be inferred from the `productClass`.
+     *
+     * @returns a new endpoint
      */
     createEndpoint(endpointName: string, accountName: string, endpointVersion?: string): Endpoint;
     /**
@@ -81,31 +125,31 @@ export declare class Client {
     docFromPath(inputPath: string): InputSource;
     /**
      * Load an input document from a base64 encoded string.
-     * @param inputString
-     * @param filename
+     * @param inputString input content, as a string.
+     * @param filename file name.
      */
     docFromBase64(inputString: string, filename: string): InputSource;
     /**
      * Load an input document from a `stream.Readable` object.
-     * @param inputStream
-     * @param filename
+     * @param inputStream input content, as a readable stream.
+     * @param filename file name.
      */
     docFromStream(inputStream: Readable, filename: string): InputSource;
     /**
      * Load an input document from a bytes string.
-     * @param inputBytes
-     * @param filename
+     * @param inputBytes input content, as readable bytes.
+     * @param filename file name.
      */
     docFromBytes(inputBytes: string, filename: string): InputSource;
     /**
      * Load an input document from a URL.
-     * @param url
+     * @param url input url. Must be HTTPS.
      */
     docFromUrl(url: string): InputSource;
     /**
      * Load an input document from a Buffer.
-     * @param buffer
-     * @param filename
+     * @param buffer input content, as a buffer.
+     * @param filename file name.
      */
     docFromBuffer(buffer: Buffer, filename: string): InputSource;
 }

package/src/client.js

@@ -4,7 +4,7 @@ var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (
     if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
     return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
 };
-var _Client_instances, _Client_buildEndpoint, _Client_initializeOTSEndpoint, _Client_cleanAccountName, _Client_getEndpoint;
+var _Client_instances, _Client_validateAsyncParams, _Client_buildEndpoint, _Client_initializeOTSEndpoint, _Client_cleanAccountName, _Client_getEndpoint;
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Client = void 0;
 const input_1 = require("./input");
@@ -14,12 +14,15 @@ const handler_1 = require("./errors/handler");
 const logger_1 = require("./logger");
 const inference_1 = require("./parsing/common/inference");
 const product_1 = require("./product");
+const promises_1 = require("node:timers/promises");
 /**
- * Mindee Client
+ * Mindee Client class that centralizes most basic operations.
+ *
+ * @category Client
  */
 class Client {
     /**
-     * @param options
+     * @param options options for the initialization of a client.
      */
     constructor({ apiKey, throwOnError, debug } = {
         apiKey: "",
@@ -37,8 +40,13 @@ class Client {
     }
     /**
      * Send a document to a synchronous endpoint and parse the predictions.
-     * @param productClass
-     * @param params
+     *
+     * @param productClass product class to use for calling the API and parsing the response.
+     * @param inputSource document to parse.
+     * @param params parameters relating to prediction options.
+     * @typeParam T an extension of an `Inference`. Can be omitted as it will be inferred from the `productClass`.
+     * @category Synchronous
+     * @returns a `Promise` containing parsing results.
      */
     async parse(productClass, inputSource, params = {
         endpoint: undefined,
@@ -60,8 +68,10 @@ class Client {
     }
     /**
      * Send the document to an asynchronous endpoint and return its ID in the queue.
-     * @param productClass
-     * @param params
+     * @param productClass product class to use for calling  the API and parsing the response.
+     * @param params parameters relating to prediction options.
+     * @category Asynchronous
+     * @returns a `Promise` containing the job (queue) corresponding to a document.
      */
     async enqueue(productClass, inputSource, params = {}) {
         const endpoint = params?.endpoint ?? __classPrivateFieldGet(this, _Client_instances, "m", _Client_initializeOTSEndpoint).call(this, productClass);
@@ -76,20 +86,87 @@ class Client {
         });
         return new common_1.AsyncPredictResponse(productClass, rawResponse.data);
     }
+    /**
+     * Polls a queue and returns its status as well as the prediction results if the parsing is done.
+     *
+     * @param productClass product class to use for calling  the API and parsing the response.
+     * @param queueId id of the queue to poll.
+     * @param params parameters relating to prediction options.
+     * @typeParam T an extension of an `Inference`. Can be omitted as it will be inferred from the `productClass`.
+     * @category Asynchronous
+     * @returns a `Promise` containing a `Job`, which also contains a `Document` if the
+     * parsing is complete.
+     */
     async parseQueued(productClass, queueId, params = {}) {
         const endpoint = params?.endpoint ?? __classPrivateFieldGet(this, _Client_instances, "m", _Client_initializeOTSEndpoint).call(this, productClass);
         const docResponse = await endpoint.getQueuedDocument(queueId);
         return new common_1.AsyncPredictResponse(productClass, docResponse.data);
     }
+    /**
+     * Send a document to an asynchronous endpoint and poll the server until the result is sent or
+     * until the maximum amount of tries is reached.
+     *
+     * @param productClass product class to use for calling the API and parsing the response.
+     * @param inputSource document to parse.
+     * @param asyncParams parameters relating to prediction options.
+     *
+     * @typeParam T an extension of an `Inference`. Can be omitted as it will be inferred from the `productClass`.
+     * @category Synchronous
+     * @returns a `Promise` containing parsing results.
+     */
+    async enqueueAndParse(productClass, inputSource, asyncParams = {
+        endpoint: undefined,
+        allWords: undefined,
+        cropper: undefined,
+        pageOptions: undefined,
+        initialDelaySec: 6,
+        delaySec: 3,
+        maxRetries: 10,
+    }) {
+        __classPrivateFieldGet(this, _Client_instances, "m", _Client_validateAsyncParams).call(this, asyncParams);
+        const enqueueResponse = await this.enqueue(productClass, inputSource, asyncParams);
+        if (enqueueResponse.job.id === undefined || enqueueResponse.job.id.length === 0) {
+            throw Error("Enqueueing of the document failed.");
+        }
+        const queueId = enqueueResponse.job.id;
+        logger_1.logger.debug(`Successfully enqueued document with job id: ${queueId}.`);
+        await (0, promises_1.setTimeout)(asyncParams.initialDelaySec * 1000);
+        let retryCounter = 1;
+        let pollResults;
+        pollResults = await this.parseQueued(productClass, queueId, asyncParams);
+        while (retryCounter < asyncParams.maxRetries) {
+            logger_1.logger.debug(`Polling server for parsing result with queueId: ${queueId}.
+Attempt n°${retryCounter}/${asyncParams.maxRetries}.
+Job status: ${pollResults.job.status}.`);
+            if (pollResults.job.status === "completed") {
+                break;
+            }
+            await (0, promises_1.setTimeout)(asyncParams.delaySec * 1000);
+            pollResults = await this.parseQueued(productClass, queueId, asyncParams);
+            retryCounter++;
+        }
+        if (pollResults.job.status !== "completed") {
+            throw Error(`Asynchronous parsing request timed out after ${asyncParams.delaySec * retryCounter} seconds`);
+        }
+        return pollResults;
+    }
+    /**
+     * Forces boolean coercion on truthy/falsy parameters.
+     * @param param input parameter to check.
+     * @returns a strict boolean value.
+     */
     getBooleanParam(param) {
         return param !== undefined ? param : false;
     }
     /**
      * Creates a custom endpoint with the given values. Raises an error if the endpoint is invalid.
-     * @param productClass Class of the product
-     * @param endpointName Name of a custom Endpoint
-     * @param accountName Name of the account tied to the active Endpoint
-     * @param version Version of a custom Endpoint
+     * @param productClass product class to use for calling the API and parsing the response.
+     * @param endpointName Name of the custom Endpoint.
+     * @param accountName Name of the account tied to the Endpoint.
+     * @param version Version of the custom Endpoint.
+     * @typeParam T an extension of an `Inference`. Can be omitted as it will be inferred from the `productClass`.
+     *
+     * @returns a new endpoint
      */
     createEndpoint(endpointName, accountName, endpointVersion) {
         const cleanAccountName = __classPrivateFieldGet(this, _Client_instances, "m", _Client_cleanAccountName).call(this, product_1.CustomV1, accountName);
@@ -117,8 +194,8 @@ class Client {
     }
     /**
      * Load an input document from a base64 encoded string.
-     * @param inputString
-     * @param filename
+     * @param inputString input content, as a string.
+     * @param filename file name.
      */
     docFromBase64(inputString, filename) {
         return new input_1.Base64Input({
@@ -128,8 +205,8 @@ class Client {
     }
     /**
      * Load an input document from a `stream.Readable` object.
-     * @param inputStream
-     * @param filename
+     * @param inputStream input content, as a readable stream.
+     * @param filename file name.
      */
     docFromStream(inputStream, filename) {
         return new input_1.StreamInput({
@@ -139,8 +216,8 @@ class Client {
     }
     /**
      * Load an input document from a bytes string.
-     * @param inputBytes
-     * @param filename
+     * @param inputBytes input content, as readable bytes.
+     * @param filename file name.
      */
     docFromBytes(inputBytes, filename) {
         return new input_1.BytesInput({
@@ -150,7 +227,7 @@ class Client {
     }
     /**
      * Load an input document from a URL.
-     * @param url
+     * @param url input url. Must be HTTPS.
      */
     docFromUrl(url) {
         return new input_1.UrlInput({
@@ -159,8 +236,8 @@ class Client {
     }
     /**
      * Load an input document from a Buffer.
-     * @param buffer
-     * @param filename
+     * @param buffer input content, as a buffer.
+     * @param filename file name.
      */
     docFromBuffer(buffer, filename) {
         return new input_1.BufferInput({
@@ -170,7 +247,17 @@ class Client {
     }
 }
 exports.Client = Client;
-_Client_instances = new WeakSet(), _Client_buildEndpoint = function _Client_buildEndpoint(endpointName, accountName, endpointVersion) {
+_Client_instances = new WeakSet(), _Client_validateAsyncParams = function _Client_validateAsyncParams(asyncParams) {
+    if (asyncParams.delaySec < 2) {
+        throw Error("Cannot set auto-parsing delay to less than 2 seconds.");
+    }
+    if (asyncParams.initialDelaySec < 4) {
+        throw Error("Cannot set initial parsing delay to less than 4 seconds.");
+    }
+    if (!Number.isInteger(asyncParams.maxRetries)) {
+        throw Error("Retry amount must be an integer.");
+    }
+}, _Client_buildEndpoint = function _Client_buildEndpoint(endpointName, accountName, endpointVersion) {
     const apiSettings = new http_1.MindeeApi({
         apiKey: this.apiKey,
         urlName: endpointName,