@ndes/iiif-images 0.1.1 → 0.2.0

package/README.md

@@ -48,11 +48,11 @@ const pipeline = await createSearchIndexer(config, {
 `representativeImage` and `associatedMedia`, so a deployment on an older schema loses
 the enrichment rather than the run.
 
-| Option          | Default         | Purpose                                                  |
-| --------------- | --------------- | -------------------------------------------------------- |
-| `storePath`     | required        | SQLite file holding resolved facts between runs          |
-| `fetchesPerRun` | 500             | Manifests, and separately `info.json` documents, per run |
-| `report`        | console warning | Called for each reference whose fetch failed             |
+| Option          | Default         | Purpose                                                    |
+| --------------- | --------------- | ---------------------------------------------------------- |
+| `storePath`     | required        | SQLite file holding resolved facts between runs            |
+| `fetchWindowMs` | no limit        | How long after construction a fetch may still start, in ms |
+| `report`        | console warning | Called for each reference whose fetch failed               |
 
 `storePath` has to be on a durable volume. The store is the only place these facts
 live between runs, and with an empty one every image is fetched from the publisher
@@ -116,12 +116,15 @@ again.
   manifest too broken to read. A 200 that is no IIIF document at all, such as an error
   page served as JSON, counts as a failure, so it is retried every run and five in a
   row stop that dataset’s manifests for the run.
-- **Budget:** `fetchesPerRun` covers all datasets together. What it defers is not
-  reported, and those works are indexed without an image until a later run.
+- **Window:** without `fetchWindowMs`, a cold store is fetched in one run, as fast
+  as each publisher keeps up. With it, a fetch that has not started once the window
+  has passed waits for a later run; one already under way finishes. The window covers
+  all datasets together, and what it defers is not reported: those works are indexed
+  without an image until a later run.
 - **Later runs:** retries, deferred works and refreshes all need the dataset to be
   reprocessed. With `PROVENANCE_FILE`, an unchanged dataset waits until it changes
-  or `PIPELINE_VERSION` is bumped, so set `fetchesPerRun` high enough to cover it in
-  one run.
+  or `PIPELINE_VERSION` is bumped, so leave `fetchWindowMs` long enough to cover it
+  in one run.
 - **Limits:** manifest requests are grouped per dataset, because manifest URLs often
   sit on a redirector such as `n2t.net`, and `info.json` requests per host. Datasets
   on one server are limited separately, with only the overall limit across them.

package/dist/lib/iiif-images.d.ts

@@ -8,14 +8,15 @@ export interface IiifImagesOptions {
      */
     readonly storePath: string;
     /**
-     * Manifests, and separately `info.json` documents, this object may fetch across
-     * all datasets, so construct one per run. Defaults to the resolver’s 500.
+     * How long after this object’s construction it may still start fetching, in
+     * milliseconds, across all datasets, so construct one per run. Unset, there is
+     * no limit: everything is fetched as fast as the publishers allow.
      *
-     * What the budget defers is not reported, and those works are indexed without an
-     * image. A cold store then warms only as datasets are reprocessed, so where
-     * unchanged datasets are skipped, set this high enough to cover them in one run.
+     * What the window defers is counted, as `manifests deferred` or
+     * `image services deferred`, but not reported, and those works are indexed
+     * without an image until a later run fetches it.
      */
-    readonly fetchesPerRun?: number;
+    readonly fetchWindowMs?: number;
     /** Told about each reference that failed. Defaults to a console warning. */
     readonly report?: (failure: ResolutionFailure) => void;
 }
@@ -24,6 +25,12 @@ export interface IiifImagesContext {
     readonly dataset: {
         readonly iri: URL;
     };
+    /**
+     * Told what became of each manifest and image service, such as
+     * `manifests fetched` or `image services from store`, so the stage can add
+     * up the run. An `@lde/pipeline` reader passes its own.
+     */
+    readonly count?: (name: string) => void;
 }
 /** A transform attachable to a reader as an `@lde/pipeline` `QuadTransform`. */
 export type IiifImagesTransform = (quads: AsyncIterable<Quad>, context: IiifImagesContext) => AsyncIterable<Quad>;

package/dist/lib/iiif-images.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"iiif-images.d.ts","sourceRoot":"","sources":["../../src/lib/iiif-images.ts"],"names":[],"mappings":"AAWA,OAAO,EAKL,KAAK,iBAAiB,EAEvB,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EACV,QAAQ,EAER,YAAY,EAEb,MAAM,aAAa,CAAC;AAErB,OAAO,KAAK,EAAE,IAAI,EAA6B,MAAM,cAAc,CAAC;AASpE,MAAM,WAAW,iBAAiB;IAChC;;;OAGG;IACH,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B;;;;;;;OAOG;IACH,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC,4EAA4E;IAC5E,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,iBAAiB,KAAK,IAAI,CAAC;CACxD;AAED,kEAAkE;AAClE,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,OAAO,EAAE;QAAE,QAAQ,CAAC,GAAG,EAAE,GAAG,CAAA;KAAE,CAAC;CACzC;AAED,gFAAgF;AAChF,MAAM,MAAM,mBAAmB,GAAG,CAChC,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,EAC1B,OAAO,EAAE,iBAAiB,KACvB,aAAa,CAAC,IAAI,CAAC,CAAC;AAEzB,MAAM,WAAW,UAAU;IACzB;;;;;OAKG;IACH,YAAY,CAAC,QAAQ,EAAE,QAAQ,GAAG,mBAAmB,GAAG,SAAS,CAAC;IAClE,4EAA4E;IAC5E,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAED;;;;;;;;;GASG;AACH,wBAAgB,UAAU,CACxB,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,iBAAiB,GACzB,UAAU,CAoHZ"}
+{"version":3,"file":"iiif-images.d.ts","sourceRoot":"","sources":["../../src/lib/iiif-images.ts"],"names":[],"mappings":"AAWA,OAAO,EAKL,KAAK,iBAAiB,EAGvB,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EACV,QAAQ,EAER,YAAY,EAEb,MAAM,aAAa,CAAC;AAErB,OAAO,KAAK,EAAE,IAAI,EAA6B,MAAM,cAAc,CAAC;AASpE,MAAM,WAAW,iBAAiB;IAChC;;;OAGG;IACH,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B;;;;;;;;OAQG;IACH,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC,4EAA4E;IAC5E,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,iBAAiB,KAAK,IAAI,CAAC;CACxD;AAED,kEAAkE;AAClE,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,OAAO,EAAE;QAAE,QAAQ,CAAC,GAAG,EAAE,GAAG,CAAA;KAAE,CAAC;IACxC;;;;OAIG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;CACzC;AAED,gFAAgF;AAChF,MAAM,MAAM,mBAAmB,GAAG,CAChC,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,EAC1B,OAAO,EAAE,iBAAiB,KACvB,aAAa,CAAC,IAAI,CAAC,CAAC;AAEzB,MAAM,WAAW,UAAU;IACzB;;;;;OAKG;IACH,YAAY,CAAC,QAAQ,EAAE,QAAQ,GAAG,mBAAmB,GAAG,SAAS,CAAC;IAClE,4EAA4E;IAC5E,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAED;;;;;;;;;GASG;AACH,wBAAgB,UAAU,CACxB,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,iBAAiB,GACzB,UAAU,CAsHZ"}

package/dist/lib/iiif-images.js

@@ -15,9 +15,9 @@ import { iiifImage, iiifImageService, iiifImageSize, representativeImageField, }
  * in `associatedMedia`, and a failed resolution is retried on a later run.
  */
 export function iiifImages(schema, options) {
-    const limits = options.fetchesPerRun === undefined
+    const limits = options.fetchWindowMs === undefined
         ? {}
-        : { fetchesPerResolver: options.fetchesPerRun };
+        : { fetchWindowMs: options.fetchWindowMs };
     const report = options.report ??
         ((failure) => console.warn(`${failure.key}: ${failure.reason}`));
     const manifests = createResolver({
@@ -85,7 +85,7 @@ export function iiifImages(schema, options) {
                     }
                     const manifestOfWork = manifestsIn(batch, aliases);
                     const dataset = context.dataset.iri.toString();
-                    const selected = await manifests.resolveAll([...manifestOfWork.values()].map((url) => ({ url, dataset })));
+                    const selected = await manifests.resolveAll([...manifestOfWork.values()].map((url) => ({ url, dataset })), countedAs('manifests', context));
                     const imageOfWork = new Map();
                     for (const [work, manifest] of manifestOfWork) {
                         const resolution = selected.get(manifest);
@@ -93,7 +93,7 @@ export function iiifImages(schema, options) {
                             imageOfWork.set(work, resolution.fact);
                         }
                     }
-                    const resolved = await services.resolveAll([...imageOfWork.values()].flatMap(({ service }) => service === undefined ? [] : [service]));
+                    const resolved = await services.resolveAll([...imageOfWork.values()].flatMap(({ service }) => service === undefined ? [] : [service]), countedAs('image services', context));
                     for (const [work, image] of imageOfWork) {
                         const representative = representativeOf(image, resolved);
                         if (representative !== undefined) {
@@ -109,6 +109,10 @@ export function iiifImages(schema, options) {
     };
 }
 const factory = new DataFactory();
+/** Pass the resolver’s counts on to the context, prefixed with what is resolved. */
+function countedAs(prefix, { count }) {
+    return { count: (name) => count?.(`${prefix} ${name}`) };
+}
 const INTEGER = factory.namedNode('http://www.w3.org/2001/XMLSchema#integer');
 /** What the profile’s `encodingFormat` for a manifest entry contains. */
 const IIIF_PRESENTATION_CONTEXT = 'iiif.io/api/presentation';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ndes/iiif-images",
-  "version": "0.1.1",
+  "version": "0.2.0",
   "description": "Enriches SCHEMA-AP-NDE works with the dimensions and sizes of their representative IIIF image, as a quad transform for the search indexer.",
   "repository": {
     "url": "git+https://github.com/netwerk-digitaal-erfgoed/stack.git",
@@ -33,13 +33,13 @@
   "dependencies": {
     "@lde/iiif-image": "^0.2.0",
     "@lde/public-fetch": "^0.1.0",
-    "@lde/resolver": "^0.2.1",
+    "@lde/resolver": "^0.7.0",
     "@rdfjs/types": "^2.0.1",
     "rdf-data-factory": "^2.1.0",
     "tslib": "^2.3.0"
   },
   "devDependencies": {
-    "@lde/pipeline": "^0.36.7",
+    "@lde/pipeline": "^0.42.0",
     "@lde/search": "^0.36.3"
   },
   "peerDependencies": {