@pepys/ingest 0.1.1 → 0.3.0

package/README.md

@@ -22,6 +22,22 @@ delivery, and appends the facts it reads to the fact log.
 bun add @pepys/ingest
 ```
 
+Serve a deployment's routes from an entry file, with the S2 settings and each secret its routes
+name:
+
+```ts
+// serve-ingest.ts
+import { serveIngest } from '@pepys/ingest';
+import { deployment } from './deployment.ts';
+
+await serveIngest({ deployment, environment: process.env });
+```
+
+```sh
+export PEPYS_S2_ACCESS_TOKEN  # the S2 access token, set in your shell beforehand
+PEPYS_S2_ENDPOINT=http://127.0.0.1:8090 PEPYS_S2_BASIN=pepys bun serve-ingest.ts
+```
+
 ## Features
 
 - `serveIngest` and `createIngest`: the receive routes of a deployment.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pepys/ingest",
-  "version": "0.1.1",
+  "version": "0.3.0",
   "description": "pepys ingest: the receive routes, the fact log and the blob store",
   "license": "MIT",
   "repository": {
@@ -22,8 +22,8 @@
   },
   "dependencies": {
     "@impz/client": "6.12.0",
-    "@pepys/kit": "0.1.1",
-    "@pepys/schema": "0.1.1",
+    "@pepys/kit": "0.3.0",
+    "@pepys/schema": "0.3.0",
     "@s2-dev/streamstore": "0.30.0",
     "@zgeoff/std": "0.4.3",
     "tiny-invariant": "1.3.3",

package/src/pull/make-pull-runner.ts

@@ -30,11 +30,12 @@ interface PullRunnerDependencies {
   append: BatchAppender;
   turns: StreamTurns;
   resyncMarks: ResyncMarks;
+  now: () => number;
 }
 
-// One run of one pull path: the pull, the envelope check, the appends, and then, for a resync pull,
-// the removal of every mark it was given. A pull's fact is noted in no mark: a resynced fact ties
-// with the body it replaces, and would mark its entity again on every run.
+// For a resync pull, removes each mark it was given that no tie set again since. A pull's fact is
+// noted in no mark: a resynced fact ties with the body it replaces, and would mark its entity again
+// on every run.
 export function makePullRunner(dependencies: PullRunnerDependencies): PullRunner {
   return async (sourceName, pathName, cursor) => {
     const source = dependencies.deployment.sources.find(
@@ -54,7 +55,7 @@ export function makePullRunner(dependencies: PullRunnerDependencies): PullRunner
       env[name] = dependencies.secrets[name] ?? '';
     }
 
-    const pulled = await path.pull(cursor, { env, marks });
+    const pulled = await path.pull(cursor, { env, marks, now: dependencies.now });
 
     if (pulled.kind !== 'batch') {
       return pulled;
@@ -85,7 +86,7 @@ export function makePullRunner(dependencies: PullRunnerDependencies): PullRunner
     }
 
     for (const mark of marks) {
-      dependencies.resyncMarks.remove(source.name, mark.entity);
+      dependencies.resyncMarks.remove(source.name, mark);
     }
 
     return {

package/src/receive/create-ingest.ts

@@ -98,6 +98,7 @@ export function createIngest(options: IngestOptions): Ingest {
       append,
       turns,
       resyncMarks: resync.marks,
+      now: options.now,
     }),
   };
 }

package/src/receive/resync-marks/create-resync-marks.ts

@@ -8,7 +8,9 @@ import type { LastBody, StoredFactTime } from './collect-tied-entities.ts';
 // it. In memory only, so a restart loses them.
 export interface ResyncMarks {
   list: (source: string) => readonly ResyncMark[];
-  remove: (source: string, entity: EntityID) => void;
+
+  // A mark set again while the pull ran is a new object, so it stays for the next pull.
+  remove: (source: string, mark: ResyncMark) => void;
 }
 
 // What the receive route of one source tells the marks.
@@ -20,8 +22,7 @@ export interface SourceMarks {
 interface SourceState {
   lastBodies: Map<EntityID, LastBody>;
 
-  // each marked entity, with the locator of the last fact stored for it
-  marks: Map<EntityID, string | undefined>;
+  marks: Map<EntityID, ResyncMark>;
 }
 
 export interface ResyncMarkStore {
@@ -53,10 +54,13 @@ export function createResyncMarks(options: ResyncMarksOptions): ResyncMarkStore
 
   return {
     marks: {
-      list: (source) =>
-        [...(sources.get(source)?.marks ?? [])].map(([entity, locator]) => ({ entity, locator })),
-      remove: (source, entity) => {
-        sources.get(source)?.marks.delete(entity);
+      list: (source) => [...(sources.get(source)?.marks.values() ?? [])],
+      remove: (source, mark) => {
+        const marks = sources.get(source)?.marks;
+
+        if (marks?.get(mark.entity) === mark) {
+          marks.delete(mark.entity);
+        }
       },
     },
     createSourceMarks: (source) => {
@@ -82,15 +86,19 @@ export function createResyncMarks(options: ResyncMarksOptions): ResyncMarkStore
   };
 }
 
-// The locator comes from the last body stored for the entity, which the caller has noted first.
+// The locator and digest come from the last body stored for the entity, which the caller has noted
+// first.
+// Each call makes a new mark object, which `remove` tells apart from the one a pull was given.
 function addMarks(
   state: SourceState,
   entities: readonly EntityID[],
   options: ResyncMarksOptions,
 ): void {
   for (const entity of entities) {
+    const last = state.lastBodies.get(entity);
+
     state.marks.delete(entity);
-    state.marks.set(entity, state.lastBodies.get(entity)?.locator);
+    state.marks.set(entity, { entity, locator: last?.locator, lastBodyDigest: last?.digest });
   }
 
   trimToCapacity(state.marks, options.capacity);
@@ -98,7 +106,7 @@ function addMarks(
 
 // drops the oldest entries first: a Map and a Set keep insertion order
 function trimToCapacity(
-  entries: Map<EntityID, LastBody> | Map<EntityID, string | undefined>,
+  entries: Map<EntityID, LastBody> | Map<EntityID, ResyncMark>,
   capacity: number,
 ): void {
   for (const key of entries.keys()) {