@ndes/iiif-images 0.1.2 → 0.2.0

package/README.md

@@ -48,11 +48,11 @@ const pipeline = await createSearchIndexer(config, {
 `representativeImage` and `associatedMedia`, so a deployment on an older schema loses
 the enrichment rather than the run.
 
-| Option          | Default         | Purpose                                                  |
-| --------------- | --------------- | -------------------------------------------------------- |
-| `storePath`     | required        | SQLite file holding resolved facts between runs          |
-| `fetchesPerRun` | 500             | Manifests, and separately `info.json` documents, per run |
-| `report`        | console warning | Called for each reference whose fetch failed             |
+| Option          | Default         | Purpose                                                    |
+| --------------- | --------------- | ---------------------------------------------------------- |
+| `storePath`     | required        | SQLite file holding resolved facts between runs            |
+| `fetchWindowMs` | no limit        | How long after construction a fetch may still start, in ms |
+| `report`        | console warning | Called for each reference whose fetch failed               |
 
 `storePath` has to be on a durable volume. The store is the only place these facts
 live between runs, and with an empty one every image is fetched from the publisher
@@ -116,12 +116,15 @@ again.
   manifest too broken to read. A 200 that is no IIIF document at all, such as an error
   page served as JSON, counts as a failure, so it is retried every run and five in a
   row stop that dataset’s manifests for the run.
-- **Budget:** `fetchesPerRun` covers all datasets together. What it defers is not
-  reported, and those works are indexed without an image until a later run.
+- **Window:** without `fetchWindowMs`, a cold store is fetched in one run, as fast
+  as each publisher keeps up. With it, a fetch that has not started once the window
+  has passed waits for a later run; one already under way finishes. The window covers
+  all datasets together, and what it defers is not reported: those works are indexed
+  without an image until a later run.
 - **Later runs:** retries, deferred works and refreshes all need the dataset to be
   reprocessed. With `PROVENANCE_FILE`, an unchanged dataset waits until it changes
-  or `PIPELINE_VERSION` is bumped, so set `fetchesPerRun` high enough to cover it in
-  one run.
+  or `PIPELINE_VERSION` is bumped, so leave `fetchWindowMs` long enough to cover it
+  in one run.
 - **Limits:** manifest requests are grouped per dataset, because manifest URLs often
   sit on a redirector such as `n2t.net`, and `info.json` requests per host. Datasets
   on one server are limited separately, with only the overall limit across them.

package/dist/lib/iiif-images.d.ts

@@ -8,15 +8,15 @@ export interface IiifImagesOptions {
      */
     readonly storePath: string;
     /**
-     * Manifests, and separately `info.json` documents, this object may fetch across
-     * all datasets, so construct one per run. Defaults to the resolver’s 500.
+     * How long after this object’s construction it may still start fetching, in
+     * milliseconds, across all datasets, so construct one per run. Unset, there is
+     * no limit: everything is fetched as fast as the publishers allow.
      *
-     * What the budget defers is counted, as `manifests deferred` or
+     * What the window defers is counted, as `manifests deferred` or
      * `image services deferred`, but not reported, and those works are indexed
-     * without an image. A cold store then warms only as datasets are reprocessed, so where
-     * unchanged datasets are skipped, set this high enough to cover them in one run.
+     * without an image until a later run fetches it.
      */
-    readonly fetchesPerRun?: number;
+    readonly fetchWindowMs?: number;
     /** Told about each reference that failed. Defaults to a console warning. */
     readonly report?: (failure: ResolutionFailure) => void;
 }

package/dist/lib/iiif-images.js

@@ -15,9 +15,9 @@ import { iiifImage, iiifImageService, iiifImageSize, representativeImageField, }
  * in `associatedMedia`, and a failed resolution is retried on a later run.
  */
 export function iiifImages(schema, options) {
-    const limits = options.fetchesPerRun === undefined
+    const limits = options.fetchWindowMs === undefined
         ? {}
-        : { fetchesPerResolver: options.fetchesPerRun };
+        : { fetchWindowMs: options.fetchWindowMs };
     const report = options.report ??
         ((failure) => console.warn(`${failure.key}: ${failure.reason}`));
     const manifests = createResolver({

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ndes/iiif-images",
-  "version": "0.1.2",
+  "version": "0.2.0",
   "description": "Enriches SCHEMA-AP-NDE works with the dimensions and sizes of their representative IIIF image, as a quad transform for the search indexer.",
   "repository": {
     "url": "git+https://github.com/netwerk-digitaal-erfgoed/stack.git",
@@ -33,13 +33,13 @@
   "dependencies": {
     "@lde/iiif-image": "^0.2.0",
     "@lde/public-fetch": "^0.1.0",
-    "@lde/resolver": "^0.4.0",
+    "@lde/resolver": "^0.7.0",
     "@rdfjs/types": "^2.0.1",
     "rdf-data-factory": "^2.1.0",
     "tslib": "^2.3.0"
   },
   "devDependencies": {
-    "@lde/pipeline": "^0.45.0",
+    "@lde/pipeline": "^0.42.0",
     "@lde/search": "^0.36.3"
   },
   "peerDependencies": {