@lde/pipeline-console-reporter 0.2.0

package/README.md

@@ -0,0 +1,17 @@
+# Pipeline Console Reporter
+
+Console progress reporter for [@lde/pipeline](../pipeline). Displays real-time pipeline progress with spinners, colours and timing information.
+
+## Usage
+
+```typescript
+import { Pipeline } from '@lde/pipeline';
+import { ConsoleReporter } from '@lde/pipeline-console-reporter';
+
+await new Pipeline({
+  datasetSelector: selector,
+  stages,
+  writers,
+  reporter: new ConsoleReporter(),
+}).run();
+```

package/dist/consoleReporter.d.ts

@@ -0,0 +1,33 @@
+import type { Dataset, Distribution } from '@lde/dataset';
+import type { DistributionAnalysisResult, ProgressReporter } from '@lde/pipeline';
+export declare class ConsoleReporter implements ProgressReporter {
+    private stageSpinner?;
+    private datasetStartTime;
+    private datasetTotal;
+    private datasetIndex;
+    private analysisResults;
+    pipelineStart(_name: string): void;
+    datasetsSelected(count: number): void;
+    datasetStart(dataset: Dataset): void;
+    distributionsAnalyzed(_dataset: Dataset, results: DistributionAnalysisResult[]): void;
+    distributionSelected(_dataset: Dataset, distribution: Distribution, importedFrom?: Distribution, importDuration?: number): void;
+    stageStart(stage: string): void;
+    stageProgress(update: {
+        elementsProcessed: number;
+        quadsGenerated: number;
+    }): void;
+    stageComplete(_stage: string, result: {
+        elementsProcessed: number;
+        quadsGenerated: number;
+        duration: number;
+    }): void;
+    stageFailed(_stage: string, error: Error): void;
+    stageSkipped(_stage: string, reason: string): void;
+    datasetComplete(_dataset: Dataset): void;
+    datasetSkipped(_dataset: Dataset, reason: string): void;
+    private printAnalysisResults;
+    pipelineComplete(result: {
+        duration: number;
+    }): void;
+}
+//# sourceMappingURL=consoleReporter.d.ts.map

package/dist/consoleReporter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"consoleReporter.d.ts","sourceRoot":"","sources":["../src/consoleReporter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC1D,OAAO,KAAK,EACV,0BAA0B,EAC1B,gBAAgB,EACjB,MAAM,eAAe,CAAC;AAKvB,qBAAa,eAAgB,YAAW,gBAAgB;IACtD,OAAO,CAAC,YAAY,CAAC,CAAM;IAC3B,OAAO,CAAC,gBAAgB,CAAK;IAC7B,OAAO,CAAC,YAAY,CAAK;IACzB,OAAO,CAAC,YAAY,CAAK;IACzB,OAAO,CAAC,eAAe,CAAoC;IAE3D,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAOlC,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAOrC,YAAY,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAYpC,qBAAqB,CACnB,QAAQ,EAAE,OAAO,EACjB,OAAO,EAAE,0BAA0B,EAAE,GACpC,IAAI;IAIP,oBAAoB,CAClB,QAAQ,EAAE,OAAO,EACjB,YAAY,EAAE,YAAY,EAC1B,YAAY,CAAC,EAAE,YAAY,EAC3B,cAAc,CAAC,EAAE,MAAM,GACtB,IAAI;IAIP,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAK/B,aAAa,CAAC,MAAM,EAAE;QACpB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,cAAc,EAAE,MAAM,CAAC;KACxB,GAAG,IAAI;IAMR,aAAa,CACX,MAAM,EAAE,MAAM,EACd,MAAM,EAAE;QACN,iBAAiB,EAAE,MAAM,CAAC;QAC1B,cAAc,EAAE,MAAM,CAAC;QACvB,QAAQ,EAAE,MAAM,CAAC;KAClB,GACA,IAAI;IAQP,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG,IAAI;IAQ/C,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI;IAQlD,eAAe,CAAC,QAAQ,EAAE,OAAO,GAAG,IAAI;IAQxC,cAAc,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI;IASvD,OAAO,CAAC,oBAAoB;IAwD5B,gBAAgB,CAAC,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;CAKrD"}

package/dist/consoleReporter.js

@@ -0,0 +1,133 @@
+import chalk from 'chalk';
+import ora from 'ora';
+import prettyMilliseconds from 'pretty-ms';
+export class ConsoleReporter {
+    stageSpinner;
+    datasetStartTime = 0;
+    datasetTotal = 0;
+    datasetIndex = 0;
+    analysisResults = [];
+    pipelineStart(_name) {
+        this.stageSpinner = ora({
+            discardStdin: false,
+            text: 'Selecting datasets',
+        }).start();
+    }
+    datasetsSelected(count) {
+        this.datasetTotal = count;
+        if (this.stageSpinner) {
+            this.stageSpinner.text = `Selected datasets: found ${chalk.bold(count)} datasets`;
+        }
+    }
+    datasetStart(dataset) {
+        this.stageSpinner?.succeed();
+        this.stageSpinner = undefined;
+        this.datasetStartTime = Date.now();
+        this.datasetIndex++;
+        const counter = this.datasetTotal
+            ? ` ${chalk.dim(`[${this.datasetIndex}/${this.datasetTotal}]`)}`
+            : '';
+        console.info();
+        console.info(`Dataset ${chalk.bold(dataset.iri.toString())}${counter}`);
+    }
+    distributionsAnalyzed(_dataset, results) {
+        this.analysisResults = results;
+    }
+    distributionSelected(_dataset, distribution, importedFrom, importDuration) {
+        this.printAnalysisResults(distribution, importedFrom, importDuration);
+    }
+    stageStart(stage) {
+        this.stageSpinner = ora({ discardStdin: false }).start();
+        this.stageSpinner.text = `Stage ${chalk.bold(stage)}`;
+    }
+    stageProgress(update) {
+        if (this.stageSpinner) {
+            this.stageSpinner.suffixText = `${update.elementsProcessed} elements, ${update.quadsGenerated} quads`;
+        }
+    }
+    stageComplete(_stage, result) {
+        if (this.stageSpinner) {
+            this.stageSpinner.suffixText = `took ${chalk.bold(prettyMilliseconds(result.duration))}`;
+            this.stageSpinner.succeed();
+            this.stageSpinner = undefined;
+        }
+    }
+    stageFailed(_stage, error) {
+        if (this.stageSpinner) {
+            this.stageSpinner.suffixText = chalk.red(error.message);
+            this.stageSpinner.fail();
+            this.stageSpinner = undefined;
+        }
+    }
+    stageSkipped(_stage, reason) {
+        if (this.stageSpinner) {
+            this.stageSpinner.suffixText = `skipped: ${chalk.red(reason)}`;
+            this.stageSpinner.fail();
+            this.stageSpinner = undefined;
+        }
+    }
+    datasetComplete(_dataset) {
+        const s = ora({
+            discardStdin: false,
+            text: `Completed in ${chalk.bold(prettyMilliseconds(Date.now() - this.datasetStartTime))}`,
+        }).start();
+        s.succeed();
+    }
+    datasetSkipped(_dataset, reason) {
+        this.printAnalysisResults();
+        const s = ora({
+            discardStdin: false,
+            text: `Skipped: ${chalk.red(reason)}`,
+        }).start();
+        s.fail();
+    }
+    printAnalysisResults(selected, importedFrom, importDuration) {
+        // Match by selected distribution URL, or by importedFrom URL (when a data
+        // dump was imported to a local SPARQL endpoint, the selected distribution
+        // is the local endpoint which doesn't appear in probe results).
+        const selectedUrl = selected?.accessUrl.toString();
+        const importedFromUrl = importedFrom?.accessUrl.toString();
+        for (const result of this.analysisResults) {
+            const resultUrl = result.distribution.accessUrl.toString();
+            const isSelected = selected &&
+                (resultUrl === selectedUrl || resultUrl === importedFromUrl);
+            const typeLabel = result.type === 'sparql'
+                ? 'SPARQL endpoint'
+                : result.type === 'data-dump'
+                    ? 'Data dump'
+                    : 'Network error';
+            const url = chalk.underline(resultUrl);
+            const s = ora({ discardStdin: false });
+            if (isSelected) {
+                if (importedFrom) {
+                    const duration = importDuration !== undefined
+                        ? ` in ${chalk.bold(prettyMilliseconds(importDuration))}`
+                        : '';
+                    s.start(`Imported ${url} (to ${chalk.underline(selectedUrl)})${duration}`);
+                }
+                else {
+                    s.start(`${typeLabel} ${url}`);
+                }
+                s.succeed();
+            }
+            else if (result.available) {
+                const detail = result.statusCode !== undefined ? ` (HTTP ${result.statusCode})` : '';
+                s.start(`${typeLabel} ${url}${detail}`);
+                s.succeed();
+            }
+            else {
+                const detail = result.error
+                    ? ` (${result.error})`
+                    : result.statusCode !== undefined
+                        ? ` (HTTP ${result.statusCode})`
+                        : '';
+                s.start(`${typeLabel} ${url}${detail}`);
+                s.fail();
+            }
+        }
+        this.analysisResults = [];
+    }
+    pipelineComplete(result) {
+        console.info(`\nPipeline completed in ${chalk.bold(prettyMilliseconds(result.duration))}`);
+    }
+}

package/dist/index.d.ts

@@ -0,0 +1,2 @@
+export { ConsoleReporter } from './consoleReporter.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC"}

package/dist/index.js

@@ -0,0 +1 @@
+export { ConsoleReporter } from './consoleReporter.js';

package/package.json

@@ -0,0 +1,34 @@
+{
+  "name": "@lde/pipeline-console-reporter",
+  "version": "0.2.0",
+  "description": "Console progress reporter for @lde/pipeline",
+  "repository": {
+    "url": "https://github.com/ldengine/lde",
+    "directory": "packages/pipeline-console-reporter"
+  },
+  "type": "module",
+  "exports": {
+    "./package.json": "./package.json",
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "development": "./src/index.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "main": "./dist/index.js",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "files": [
+    "dist",
+    "!**/*.tsbuildinfo"
+  ],
+  "dependencies": {
+    "@lde/dataset": "0.7.0",
+    "@lde/pipeline": "0.13.0",
+    "chalk": "^5.4.1",
+    "ora": "^8.2.0",
+    "pretty-ms": "^9.3.0",
+    "tslib": "^2.3.0"
+  }
+}