@artian-techs/adonis-orbit 0.1.1 → 0.2.0

package/README.md

@@ -45,10 +45,16 @@ Under it, one button reads the page in the light or in the dark. The page reads
 the mode of the machine while nobody chose; a choice is held in a cookie. The
 page holds no JavaScript.
 
+The list carries a pager over the table and under it: the number of every page
+that stands near the one that is read, the first, the last, and a gap between
+them. **Rows** says how many jobs one page holds — 25, 50 or 100. The choice is
+held in a cookie, thus the menu carries it from one status to the next. Without
+one, fifty.
+
 | Entry | What it shows |
 | --- | --- |
 | Overview | four numbers, then the last jobs of every status |
-| Pending, Running, Retrying, Completed, Failed | the same list, of one status |
+| Pending, Running, Retrying, Completed, Failed, Lost | the same list, of one status |
 | Silenced | the classes that the lists leave out |
 | Metrics | what each class and each queue cost over the last 24 hours |
 
@@ -203,6 +209,7 @@ One row for one job, completed at each step.
 | `retrying` | a try failed, the next one is set |
 | `completed` | done |
 | `failed` | every try fell |
+| `lost` | the queue let the job go, and no worker said how it ended |
 
 The row that the dispatch writes and the row that the worker writes are the
 same row. Either of the two can arrive first, thus the first one writes and the
@@ -210,8 +217,8 @@ second one completes.
 
 ## Ask a job again
 
-From the page, with the button on a row that failed. Or with the job that the
-package carries:
+From the page, with the button on a row that failed or that the queue lost. Or
+with the job that the package carries:
 
 ```ts
 import { JobRun, JobRunStatus, retryRun } from '@artian-techs/adonis-orbit'
@@ -261,6 +268,44 @@ them away at the end.
 every status to read and the name `orbit` and the word `orbit`. It reads a
 database of its own, `orbit_preview`, and makes it if it is not there.
 
+## Close the rows that wait for nothing
+
+A worker publishes what it does, and Orbit reads it. A worker that refuses a
+job **before it starts** publishes nothing: it fails the job in the queue and
+takes the next one. This happens at every deployment where the worker restarts
+after the API, because the worker of the old build does not carry the class
+that the new API asks for.
+
+The row of that job stays `pending`, and the page shows a job that waits for a
+worker that will never take it.
+
+`ReconcileJobRuns` asks the queue about every row that stayed open too long:
+
+```ts
+import { ReconcileJobRuns } from '@artian-techs/adonis-orbit'
+
+await ReconcileJobRuns.schedule({ minutes: 5 }).cron('* * * * *').run()
+```
+
+- the queue holds the job, thus the row is right and stays as it is;
+- the queue holds the answer, thus the row takes it, with its error;
+- the queue holds nothing, thus the row becomes `lost`.
+
+`minutes` says how long a row stays open before the queue is asked about it.
+It leaves the events of the worker the time to land: a row that a worker closed
+a second ago is never read. Without it, five minutes.
+
+The rows are read from the table, thus this needs `DatabaseSink`. The answers
+go to every sink of the configuration, as the events of the worker do.
+
+A queue that keeps its failed jobs gives the error that stopped them. Ask for
+it in `config/queue.ts`, else the queue removes a job as soon as it fails and
+the row can only say `lost`:
+
+```ts
+defaultJobOptions: { removeOnFail: false },
+```
+
 ## Take out the traces that ended well
 
 The package carries `PruneJobRuns` and names it to the queue itself. Give it a

package/build/index.d.ts

@@ -5,6 +5,8 @@ export type { Authorize, JobEvent, JobIdentity, JobOutcome, OrbitConfig, Payload
 export { default as JobRun, JobRunStatus, type JobRunStatusValue } from './src/models/job_run.js';
 export { default as SilencedJob } from './src/models/silenced_job.js';
 export { default as PruneJobRuns } from './src/jobs/prune_job_runs.js';
+export { default as ReconcileJobRuns } from './src/jobs/reconcile_job_runs.js';
+export { reconcileJobRuns, type QueueLookup, type ReconcileOptions } from './src/reconcile.js';
 export { retryRun } from './src/retry.js';
 export { stubsRoot } from './stubs/main.js';
 export { configure } from './configure.js';

package/build/index.js

@@ -4,6 +4,8 @@ export { default as DatabaseSink } from './src/sinks/database_sink.js';
 export { default as JobRun, JobRunStatus } from './src/models/job_run.js';
 export { default as SilencedJob } from './src/models/silenced_job.js';
 export { default as PruneJobRuns } from './src/jobs/prune_job_runs.js';
+export { default as ReconcileJobRuns } from './src/jobs/reconcile_job_runs.js';
+export { reconcileJobRuns } from './src/reconcile.js';
 export { retryRun } from './src/retry.js';
 export { stubsRoot } from './stubs/main.js';
 export { configure } from './configure.js';

package/build/providers/orbit_provider.js

@@ -1,6 +1,7 @@
 import { Locator } from '@adonisjs/queue';
 import edge from 'edge.js';
 import PruneJobRuns from '../src/jobs/prune_job_runs.js';
+import ReconcileJobRuns from '../src/jobs/reconcile_job_runs.js';
 const DEFAULTS = { path: '/queue', authorize: () => false, sinks: [] };
 /**
  * Watch what the queue does, and keep it.
@@ -26,8 +27,17 @@ export default class OrbitProvider {
          * lives here. It names itself to the queue.
          */
         Locator.register(PruneJobRuns.name, PruneJobRuns);
+        Locator.register(ReconcileJobRuns.name, ReconcileJobRuns);
         const { recordJobRuns } = await import('../src/recorder.js');
-        recordJobRuns(config);
+        const recorder = recordJobRuns(config);
+        /**
+         * A sink answers after the job does, and the application closes its
+         * database when it stops. The writes that are open are waited for first.
+         *
+         * This hook runs before the providers stop, thus the database of the
+         * application is still open.
+         */
+        this.app.terminating(() => recorder.flush());
         const router = await this.app.container.make('router');
         const { registerRoutes } = await import('../src/routes.js');
         registerRoutes(router, config);

package/build/src/controllers/dashboard_controller.d.ts

@@ -5,6 +5,10 @@ import type { HttpContext } from '@adonisjs/core/http';
  * It reads the table, not Redis: a job that ended left Redis, and the table
  * keeps it. The page is served by the API itself, outside the front, thus it
  * answers even when the front is down.
+ *
+ * A button answers with the page that carried it, and that page keeps the
+ * values of its address: a list of one status stays on that status, and a page
+ * of a pager stays on its number.
  */
 export default class DashboardController {
     #private;
@@ -16,6 +20,13 @@ export default class DashboardController {
      * the other, the page shows the numbers and the last jobs of every status.
      */
     index(ctx: HttpContext): Promise<void>;
+    /**
+     * Read more rows on one page, or fewer.
+     *
+     * The choice is held in a cookie, as the mode is. A page that stands past
+     * the last one after the change answers with the last one.
+     */
+    perPage({ params, response }: HttpContext): Promise<void>;
     /**
      * The classes that the lists leave out.
      *

package/build/src/controllers/dashboard_controller.js

@@ -7,11 +7,20 @@ import JobRun, { JobRunStatus } from '../models/job_run.js';
 import SilencedJob from '../models/silenced_job.js';
 import { retryRun } from '../retry.js';
 const PER_PAGE = 50;
+/** The sizes of a page that a person chooses between. */
+const PER_PAGE_SIZES = [25, 50, 100];
+/** Where the size of a page is held, and for how long. */
+const PER_PAGE_COOKIE = 'orbit_per_page';
+const PER_PAGE_AGE = '1y';
+/** How many numbers of pages stand on each side of the one that is read. */
+const PAGER_WINDOW = 2;
 /** The two modes that a person chooses between. */
 const THEMES = ['light', 'dark'];
 /** Where the choice of the mode is held, and for how long. */
 const THEME_COOKIE = 'orbit_theme';
 const THEME_AGE = '1y';
+/** The statuses of a job that a person may ask for again. */
+const ENDED_BADLY = [JobRunStatus.FAILED, JobRunStatus.LOST];
 /** How far back the metrics read. */
 const METRICS_WINDOW = { hours: 24 };
 const STATUS_LABELS = {
@@ -20,6 +29,7 @@ const STATUS_LABELS = {
     [JobRunStatus.RETRYING]: 'Retrying',
     [JobRunStatus.COMPLETED]: 'Completed',
     [JobRunStatus.FAILED]: 'Failed',
+    [JobRunStatus.LOST]: 'Lost',
 };
 /**
  * The drawings of the side menu. They are written here, and the page prints
@@ -32,6 +42,7 @@ const ICONS = {
     retrying: '<path d="M21 12a9 9 0 1 1-3-6.7"/><path d="M21 4v5h-5"/>',
     completed: '<circle cx="12" cy="12" r="9"/><path d="M8 12.5l3 3 5-6"/>',
     failed: '<circle cx="12" cy="12" r="9"/><path d="M12 7v6"/><path d="M12 16.5v.5"/>',
+    lost: '<circle cx="12" cy="12" r="9"/><path d="M9.5 9.5l5 5"/><path d="M14.5 9.5l-5 5"/>',
     silenced: '<path d="M18 8a6 6 0 0 0-9.3-5"/><path d="M6 9v5l-2 3h13"/><path d="M10.5 20a2 2 0 0 0 3 0"/><path d="M3 3l18 18"/>',
     metrics: '<path d="M4 20V10"/><path d="M10 20V4"/><path d="M16 20v-7"/><path d="M22 20H2"/>',
 };
@@ -41,6 +52,10 @@ const ICONS = {
  * It reads the table, not Redis: a job that ended left Redis, and the table
  * keeps it. The page is served by the API itself, outside the front, thus it
  * answers even when the front is down.
+ *
+ * A button answers with the page that carried it, and that page keeps the
+ * values of its address: a list of one status stays on that status, and a page
+ * of a pager stays on its number.
  */
 export default class DashboardController {
     #silenced;
@@ -57,21 +72,32 @@ export default class DashboardController {
         const job = request.input('job');
         const page = Number(request.input('page', 1)) || 1;
         const silenced = await this.#silencedNames();
-        const query = JobRun.query().orderBy('id', 'desc');
-        if (status) {
-            query.where('status', status);
-        }
+        const perPage = this.#perPage(request);
+        const build = () => {
+            const query = JobRun.query().orderBy('id', 'desc');
+            if (status) {
+                query.where('status', status);
+            }
+            /**
+             * A person who asks for one job asks for that job, thus a class that is
+             * silenced answers here as any other class.
+             */
+            if (job) {
+                query.where('jobId', job);
+            }
+            else if (silenced.length > 0) {
+                query.whereNotIn('name', silenced);
+            }
+            return query;
+        };
+        let runs = await build().paginate(page, perPage);
         /**
-         * A person who asks for one job asks for that job, thus a class that is
-         * silenced answers here as any other class.
+         * A page that stands past the last one holds nothing. The last page holds
+         * the rows that the person asked for.
          */
-        if (job) {
-            query.where('jobId', job);
+        if (page > runs.lastPage && runs.lastPage > 0) {
+            runs = await build().paginate(runs.lastPage, perPage);
         }
-        else if (silenced.length > 0) {
-            query.whereNotIn('name', silenced);
-        }
-        const runs = await query.paginate(page, PER_PAGE);
         const answers = await this.#answers(runs.all());
         const alone = Boolean(status || job);
         return this.#render(ctx, 'index', {
@@ -81,10 +107,89 @@ export default class DashboardController {
             stats: alone ? null : await this.#stats(),
             status: status ?? '',
             rows: runs.all().map((run) => this.#row(run, silenced, answers)),
-            page: runs.currentPage,
-            lastPage: runs.lastPage,
+            pager: this.#pager(runs, { status, job }),
+            perPage: { current: perPage, sizes: PER_PAGE_SIZES },
+        });
+    }
+    /**
+     * How many rows one page holds.
+     *
+     * The choice is held in a cookie, thus the menu carries it from one status
+     * to the next and no address holds it.
+     */
+    #perPage(request) {
+        const size = Number(request.plainCookie(PER_PAGE_COOKIE));
+        return PER_PAGE_SIZES.includes(size) ? size : PER_PAGE;
+    }
+    /**
+     * Every address that the pager leads to. The page reads them, it builds
+     * none.
+     *
+     * The numbers stand around the page that is read, and the first and the last
+     * stand at the ends. A hole between two of them is a step that the page
+     * writes as a gap.
+     */
+    #pager(runs, filters) {
+        const current = runs.currentPage;
+        const last = runs.lastPage;
+        const href = (page) => this.#href({ ...filters, page: page > 1 ? page : undefined });
+        const pager = {
+            page: current,
+            lastPage: last,
             total: runs.total,
+            previous: current > 1 ? href(current - 1) : null,
+            next: current < last ? href(current + 1) : null,
+            pages: [],
+        };
+        if (last < 2) {
+            return pager;
+        }
+        const numbers = new Set([1, last]);
+        for (let number = current - PAGER_WINDOW; number <= current + PAGER_WINDOW; number++) {
+            if (number >= 1 && number <= last) {
+                numbers.add(number);
+            }
+        }
+        let previous = 0;
+        for (const number of [...numbers].sort((a, b) => a - b)) {
+            if (previous > 0 && number - previous > 1) {
+                pager.pages.push({ number: null, href: null, current: false });
+            }
+            pager.pages.push({ number, href: href(number), current: number === current });
+            previous = number;
+        }
+        return pager;
+    }
+    /** One address of the page, with the values that name what it shows. */
+    #href(params) {
+        const query = new URLSearchParams();
+        for (const [key, value] of Object.entries(params)) {
+            if (value !== undefined && value !== '') {
+                query.set(key, String(value));
+            }
+        }
+        const search = query.toString();
+        const { path } = this.#config();
+        return search ? `${path}?${search}` : path;
+    }
+    /**
+     * Read more rows on one page, or fewer.
+     *
+     * The choice is held in a cookie, as the mode is. A page that stands past
+     * the last one after the change answers with the last one.
+     */
+    async perPage({ params, response }) {
+        const size = Number(params.count);
+        if (!PER_PAGE_SIZES.includes(size)) {
+            return response.notFound({ success: false, error: 'Not found' });
+        }
+        response.plainCookie(PER_PAGE_COOKIE, String(size), {
+            path: this.#config().path,
+            maxAge: PER_PAGE_AGE,
+            httpOnly: false,
+            sameSite: 'lax',
         });
+        return response.redirect().withQs().back();
     }
     /** What the page is called. */
     #title(status, job) {
@@ -134,8 +239,8 @@ export default class DashboardController {
             active: 'metrics',
             title: 'Metrics',
             hours: METRICS_WINDOW.hours,
-            byJob: await this.#metricsBy('name', since),
-            byQueue: await this.#metricsBy('queue', since),
+            byJob: await this.#metricsBy(['name', 'queue'], since),
+            byQueue: await this.#metricsBy(['queue'], since),
         });
     }
     /**
@@ -147,7 +252,7 @@ export default class DashboardController {
     async retry({ params, response }) {
         const run = await JobRun.findOrFail(params.id);
         await retryRun(run);
-        return response.redirect().back();
+        return response.redirect().withQs().back();
     }
     /**
      * Read the page in the light, or in the dark.
@@ -167,7 +272,7 @@ export default class DashboardController {
             httpOnly: false,
             sameSite: 'lax',
         });
-        return response.redirect().back();
+        return response.redirect().withQs().back();
     }
     /** Take a class out of the lists, or put it back. */
     async silence({ params, response }) {
@@ -179,7 +284,7 @@ export default class DashboardController {
         else {
             await SilencedJob.create({ name });
         }
-        return response.redirect().back();
+        return response.redirect().withQs().back();
     }
     /** The four numbers of the overview. */
     async #stats() {
@@ -201,23 +306,29 @@ export default class DashboardController {
         return Number(row.total);
     }
     /**
-     * What one column cost over the window.
+     * What one column, or two, cost over the window.
      *
      * The count of the failures rides with the count of the rows: one reading of
      * the table answers both.
+     *
+     * A class that runs on two queues is read twice, once for each queue: the
+     * duration of a job is the duration of the queue that carries it, and one
+     * average over the two says nothing about either.
      */
-    async #metricsBy(column, since) {
-        const result = await db.rawQuery(`select ${column} as label,
+    async #metricsBy(columns, since) {
+        const group = columns.join(', ');
+        const result = await db.rawQuery(`select ${group},
               count(*) as total,
               count(*) filter (where status = ?) as failed,
               round(avg(duration)::numeric, 1) as average,
               round(max(duration)::numeric, 1) as longest
          from ${JobRun.table}
         where finished_at > ?
-        group by ${column}
+        group by ${group}
         order by count(*) desc`, [JobRunStatus.FAILED, since]);
         return result.rows.map((row) => ({
-            label: row.label,
+            label: row[columns[0]],
+            queue: columns.length > 1 ? row[columns[1]] : null,
             total: Number(row.total),
             failed: Number(row.failed),
             average: row.average === null ? null : Number(row.average),
@@ -279,7 +390,7 @@ export default class DashboardController {
             retriedAt: run.retriedAt?.toFormat('dd LLL yyyy HH:mm:ss') ?? null,
             retriedJobId: run.retriedJobId,
             retriedStatus: answer ? STATUS_LABELS[answer] : null,
-            canRetry: run.status === JobRunStatus.FAILED && !run.retriedJobId,
+            canRetry: ENDED_BADLY.includes(run.status) && !run.retriedJobId,
             isSilenced: silenced.includes(run.name),
         };
     }

package/build/src/jobs/reconcile_job_runs.d.ts

@@ -0,0 +1,18 @@
+import { Job } from '@adonisjs/queue';
+import type { JobOptions } from '@adonisjs/queue/types';
+interface ReconcileJobRunsPayload {
+    /** How long a row stays open before the queue is asked about it, in minutes. */
+    minutes?: number;
+}
+/**
+ * Close the rows that wait for an answer that never comes.
+ *
+ * A worker that refuses a job before it starts publishes nothing. The row of
+ * that job stays open, and the page shows it as a job that waits. This asks
+ * the queue about every row that stayed open, and writes what the queue says.
+ */
+export default class ReconcileJobRuns extends Job<ReconcileJobRunsPayload> {
+    static options: JobOptions;
+    execute(): Promise<void>;
+}
+export {};

package/build/src/jobs/reconcile_job_runs.js

@@ -0,0 +1,26 @@
+import app from '@adonisjs/core/services/app';
+import logger from '@adonisjs/core/services/logger';
+import { Job } from '@adonisjs/queue';
+import { reconcileJobRuns } from '../reconcile.js';
+/**
+ * Close the rows that wait for an answer that never comes.
+ *
+ * A worker that refuses a job before it starts publishes nothing. The row of
+ * that job stays open, and the page shows it as a job that waits. This asks
+ * the queue about every row that stayed open, and writes what the queue says.
+ */
+export default class ReconcileJobRuns extends Job {
+    static options = {
+        queue: 'default',
+        maxRetries: 1,
+    };
+    async execute() {
+        const config = app.config.get('orbit', {
+            path: '/queue',
+            authorize: () => false,
+            sinks: [],
+        });
+        const events = await reconcileJobRuns(config, { minutes: this.payload.minutes });
+        logger.info(`${events.length} trace(s) of jobs closed.`);
+    }
+}

package/build/src/models/job_run.d.ts

@@ -6,6 +6,9 @@ import type { DateTime } from 'luxon';
  * `pending` and `running` are what the worker is about to do or does now.
  * `completed` and `failed` are what it did. `retrying` names a job that failed
  * a try and waits for the next one.
+ *
+ * `lost` names a job that the queue does not hold any more, and that no worker
+ * said anything about.
  */
 export declare const JobRunStatus: {
     readonly PENDING: "pending";
@@ -13,6 +16,7 @@ export declare const JobRunStatus: {
     readonly RETRYING: "retrying";
     readonly COMPLETED: "completed";
     readonly FAILED: "failed";
+    readonly LOST: "lost";
 };
 export type JobRunStatusValue = (typeof JobRunStatus)[keyof typeof JobRunStatus];
 export default class JobRun extends BaseModel {

package/build/src/models/job_run.js

@@ -11,6 +11,9 @@ import { BaseModel, column } from '@adonisjs/lucid/orm';
  * `pending` and `running` are what the worker is about to do or does now.
  * `completed` and `failed` are what it did. `retrying` names a job that failed
  * a try and waits for the next one.
+ *
+ * `lost` names a job that the queue does not hold any more, and that no worker
+ * said anything about.
  */
 export const JobRunStatus = {
     PENDING: 'pending',
@@ -18,6 +21,7 @@ export const JobRunStatus = {
     RETRYING: 'retrying',
     COMPLETED: 'completed',
     FAILED: 'failed',
+    LOST: 'lost',
 };
 export default class JobRun extends BaseModel {
 }

package/build/src/reconcile.d.ts

@@ -0,0 +1,42 @@
+import type { JobEvent, OrbitConfig } from './types.js';
+/**
+ * Who answers about one job.
+ *
+ * The queue of the application answers this, and so does a queue of a test.
+ * The words are those of a queue, and of no framework.
+ */
+export interface QueueLookup {
+    getJob(id: string, queue: string): Promise<{
+        status: string;
+        error?: string;
+    } | null>;
+}
+export interface ReconcileOptions {
+    /** How long a row stays open before the queue is asked about it, in minutes. */
+    minutes?: number;
+    /** How many rows one call reads. */
+    limit?: number;
+    /** Who answers about a job. Without one, the queue of the application. */
+    queue?: QueueLookup;
+}
+/**
+ * Ask the queue about the jobs that the table still calls open.
+ *
+ * A worker publishes what it does, and the recorder reads it. A worker that
+ * refuses a job before it starts publishes nothing: it fails the job in the
+ * queue and takes the next one. The row of that job then waits for an answer
+ * that never comes.
+ *
+ * This reads the rows that stayed open, and asks the queue about each one:
+ *
+ * - the queue holds the job, thus the row is right and stays as it is;
+ * - the queue holds the answer, thus the row takes it;
+ * - the queue holds nothing, thus the job is `lost`.
+ *
+ * The grace leaves the events of the worker the time to land. A row that a
+ * worker closed a second ago is not read here.
+ *
+ * The rows are read from the table, thus this needs `DatabaseSink`. The
+ * answers go to every sink of the configuration.
+ */
+export declare function reconcileJobRuns(config: OrbitConfig, options?: ReconcileOptions): Promise<JobEvent[]>;

package/build/src/reconcile.js

@@ -0,0 +1,81 @@
+import logger from '@adonisjs/core/services/logger';
+import { QueueManager } from '@adonisjs/queue';
+import { DateTime } from 'luxon';
+import JobRun, { JobRunStatus } from './models/job_run.js';
+/** The statuses of a row that waits for an answer. */
+const OPEN = [JobRunStatus.PENDING, JobRunStatus.RUNNING, JobRunStatus.RETRYING];
+/** The statuses that the queue gives to a job that it still holds. */
+const HELD = ['pending', 'active', 'delayed'];
+/** What a row carries when the queue lets the job go and says nothing. */
+const LOST = 'The queue does not hold this job any more. No worker said how it ended.';
+/** How long a row stays open before the queue is asked about it, in minutes. */
+const GRACE = 5;
+/** How many rows one call reads. */
+const LIMIT = 500;
+/**
+ * Ask the queue about the jobs that the table still calls open.
+ *
+ * A worker publishes what it does, and the recorder reads it. A worker that
+ * refuses a job before it starts publishes nothing: it fails the job in the
+ * queue and takes the next one. The row of that job then waits for an answer
+ * that never comes.
+ *
+ * This reads the rows that stayed open, and asks the queue about each one:
+ *
+ * - the queue holds the job, thus the row is right and stays as it is;
+ * - the queue holds the answer, thus the row takes it;
+ * - the queue holds nothing, thus the job is `lost`.
+ *
+ * The grace leaves the events of the worker the time to land. A row that a
+ * worker closed a second ago is not read here.
+ *
+ * The rows are read from the table, thus this needs `DatabaseSink`. The
+ * answers go to every sink of the configuration.
+ */
+export async function reconcileJobRuns(config, options = {}) {
+    const before = DateTime.now().minus({ minutes: options.minutes ?? GRACE });
+    const open = await JobRun.query()
+        .whereIn('status', [...OPEN])
+        .where('updatedAt', '<', before.toJSDate())
+        .orderBy('id', 'asc')
+        .limit(options.limit ?? LIMIT);
+    if (open.length === 0) {
+        return [];
+    }
+    const queue = options.queue ?? QueueManager.use();
+    const at = new Date();
+    const events = [];
+    for (const run of open) {
+        const record = await queue.getJob(run.jobId, run.queue);
+        if (record && HELD.includes(record.status)) {
+            continue;
+        }
+        const outcome = record ? record.status : 'lost';
+        events.push({
+            type: 'finished',
+            at,
+            job: {
+                id: run.jobId,
+                name: run.name,
+                queue: run.queue,
+                payload: run.payload ?? {},
+                attempts: run.attempts,
+            },
+            outcome,
+            duration: null,
+            error: record ? record.error || null : LOST,
+        });
+    }
+    if (events.length === 0) {
+        return [];
+    }
+    for (const sink of config.sinks ?? []) {
+        try {
+            await sink.write(events);
+        }
+        catch (error) {
+            logger.error({ err: error }, 'The trace of a job could not be written');
+        }
+    }
+    return events;
+}

package/build/src/recorder.d.ts

@@ -10,5 +10,10 @@ import type { OrbitConfig } from './types.js';
  * A sink that refuses an event must never stop the job that runs.
  *
  * A configuration that names no sink keeps nothing.
+ *
+ * The recorder gives back a `flush`. It waits for the writes that are open,
+ * and the application calls it before it closes what the sinks use.
  */
-export declare function recordJobRuns(config: OrbitConfig): void;
+export declare function recordJobRuns(config: OrbitConfig): {
+    flush: () => Promise<void>;
+};

package/build/src/recorder.js

@@ -47,15 +47,35 @@ function keep(rule, job, payload) {
  * A sink that refuses an event must never stop the job that runs.
  *
  * A configuration that names no sink keeps nothing.
+ *
+ * The recorder gives back a `flush`. It waits for the writes that are open,
+ * and the application calls it before it closes what the sinks use.
  */
 export function recordJobRuns(config) {
     const sinks = config.sinks ?? [];
     const rule = config.payload ?? 'full';
+    /** The writes that are started, and that no sink finished yet. */
+    const pending = new Set();
     function write(events) {
         for (const sink of sinks) {
-            void sink.write(events).catch((error) => {
+            const done = sink.write(events).catch((error) => {
                 logger.error({ err: error }, 'The trace of a job could not be written');
             });
+            pending.add(done);
+            void done.finally(() => pending.delete(done));
+        }
+    }
+    /**
+     * Wait for every write that is open.
+     *
+     * A command puts a job in the queue and stops at once. The writes of the
+     * sinks are still open at that moment, and the application closes its
+     * database after this call. A write that a sink starts while this one waits
+     * is waited for too.
+     */
+    async function flush() {
+        while (pending.size > 0) {
+            await Promise.all(pending);
         }
     }
     function identify(job, queue) {
@@ -99,4 +119,5 @@ export function recordJobRuns(config) {
             ]);
         },
     });
+    return { flush };
 }

package/build/src/routes.js

@@ -14,6 +14,7 @@ export function registerRoutes(router, config) {
         router.post('/runs/:id/retry', [DashboardController, 'retry']).as('orbit.retry');
         router.post('/jobs/:name/silence', [DashboardController, 'silence']).as('orbit.silence');
         router.post('/theme/:theme', [DashboardController, 'theme']).as('orbit.theme');
+        router.post('/per-page/:count', [DashboardController, 'perPage']).as('orbit.perPage');
     })
         .prefix(config.path)
         .use(dashboardAuth(config));

package/build/src/sinks/database_sink.js

@@ -5,6 +5,7 @@ const STATUS = {
     completed: JobRunStatus.COMPLETED,
     failed: JobRunStatus.FAILED,
     retrying: JobRunStatus.RETRYING,
+    lost: JobRunStatus.LOST,
 };
 /**
  * Keep the trace of the jobs in the table of the application.
@@ -74,7 +75,7 @@ export default class DatabaseSink {
     }
     /** Say how the try of a job ended. */
     #close({ job, at, outcome, duration, error }) {
-        const ended = outcome === 'completed' || outcome === 'failed';
+        const ended = outcome !== 'retrying';
         return JobRun.query()
             .where('jobId', job.id)
             .update({

package/build/src/types.d.ts

@@ -25,8 +25,14 @@ export interface JobIdentity {
     /** How many times a worker took the job. */
     attempts: number;
 }
-/** How a try of a job ended. */
-export type JobOutcome = 'completed' | 'failed' | 'retrying';
+/**
+ * How a try of a job ended.
+ *
+ * `lost` names a job that the queue does not hold any more, and that no worker
+ * said anything about. A worker that stops between the two says nothing, and
+ * the job would wait in the trace for ever.
+ */
+export type JobOutcome = 'completed' | 'failed' | 'retrying' | 'lost';
 /**
  * One step of one job.
  *

package/build/templates/components/layout.edge

@@ -104,6 +104,7 @@
     .retrying { background: #fef3c7; color: #92400e; }
     .completed { background: #d1fae5; color: #065f46; }
     .failed { background: #fee2e2; color: #991b1b; }
+    .lost { background: #ede9fe; color: #5b21b6; }
     .muted { color: var(--muted); font-size: 12px; }
     .actions { display: flex; gap: 6px; align-items: center; }
     .actions a { color: inherit; white-space: nowrap; }
@@ -117,8 +118,23 @@
       overflow-x: auto; font-size: 12px; max-width: 520px; white-space: pre-wrap;
     }
     details summary { cursor: pointer; color: var(--muted); font-size: 12px; }
-    .pager { display: flex; gap: 14px; align-items: center; margin-top: 16px; }
+    .pager {
+      display: flex; gap: 8px; align-items: center; flex-wrap: wrap;
+      margin: 16px 0; font-size: 13px;
+    }
     .pager a { color: inherit; }
+    .pager form { display: inline; }
+    .page {
+      display: inline-block; min-width: 30px; padding: 4px 8px; border-radius: 6px;
+      border: 1px solid var(--line); background: var(--card); text-align: center;
+      text-decoration: none; font: inherit; font-size: 13px; color: inherit; cursor: pointer;
+    }
+    a.page:hover, button.page:hover { border-color: var(--accent); color: var(--accent); }
+    .page.current {
+      border-color: var(--accent); background: var(--accent-soft);
+      color: var(--accent); font-weight: 600; cursor: default;
+    }
+    .rows { display: flex; gap: 6px; align-items: center; margin-left: auto; }
     .empty { padding: 28px; text-align: center; color: var(--muted); }
   </style>
 </head>

package/build/templates/components/pager.edge

@@ -0,0 +1,39 @@
+<div class="pager">
+  <span class="muted">{{ pager.total }} job(s)</span>
+
+  @if(pager.previous)
+    <a href="{{ pager.previous }}">← Previous</a>
+  @end
+
+  @each(entry in pager.pages)
+    @if(entry.number === null)
+      <span class="muted">…</span>
+    @elseif(entry.current)
+      <span class="page current">{{ entry.number }}</span>
+    @else
+      <a class="page" href="{{ entry.href }}">{{ entry.number }}</a>
+    @end
+  @end
+
+  @if(pager.next)
+    <a href="{{ pager.next }}">Next →</a>
+  @end
+
+  @if(rows)
+    <span class="rows">
+      <span class="muted">Rows</span>
+      @each(size in perPage.sizes)
+        @if(size === perPage.current)
+          <span class="page current">{{ size }}</span>
+        @else
+          <form method="post" action="{{ path }}/per-page/{{ size }}">
+            @if(csrfToken)
+              <input type="hidden" name="_csrf" value="{{ csrfToken }}">
+            @end
+            <button type="submit" class="page">{{ size }}</button>
+          </form>
+        @end
+      @end
+    </span>
+  @end
+</div>

package/build/templates/index.edge

@@ -22,6 +22,8 @@
     <h2>Recent jobs</h2>
   @end
 
+  @!component('orbit::components/pager', { pager, path, csrfToken, perPage, rows: true })
+
   <div class="panel">
     <table>
       <thead>
@@ -99,13 +101,5 @@
     </table>
   </div>
 
-  <div class="pager">
-    @if(page > 1)
-      <a href="{{ path }}?page={{ page - 1 }}&status={{ status }}">← Previous</a>
-    @end
-    <span class="muted">Page {{ page }} of {{ lastPage }} — {{ total }} job(s)</span>
-    @if(page < lastPage)
-      <a href="{{ path }}?page={{ page + 1 }}&status={{ status }}">Next →</a>
-    @end
-  </div>
+  @!component('orbit::components/pager', { pager, path, csrfToken, perPage, rows: true })
 @end

package/build/templates/metrics.edge

@@ -7,6 +7,7 @@
       <thead>
         <tr>
           <th>Job</th>
+          <th>Queue</th>
           <th>Ended</th>
           <th>Failed</th>
           <th>Average</th>
@@ -17,13 +18,14 @@
         @each(row in byJob)
           <tr>
             <td><b>{{ row.label }}</b></td>
+            <td>{{ row.queue }}</td>
             <td>{{ row.total }}</td>
             <td>{{ row.failed }}</td>
             <td>{{ row.average === null ? '—' : row.average + ' ms' }}</td>
             <td>{{ row.longest === null ? '—' : row.longest + ' ms' }}</td>
           </tr>
         @else
-          <tr><td colspan="5" class="empty">No job ended in this window.</td></tr>
+          <tr><td colspan="6" class="empty">No job ended in this window.</td></tr>
         @end
       </tbody>
     </table>

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@artian-techs/adonis-orbit",
-  "version": "0.1.1",
+  "version": "0.2.0",
   "type": "module",
   "description": "See what the queue of AdonisJS does, keep it, and ask a job again.",
   "main": "build/index.js",