@tinyshed/tinystore 0.1.0-rc.7 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/README.md +2 -2
  2. package/dist/blobs.d.ts +1 -1
  3. package/dist/blobs.js +13 -5
  4. package/dist/cancel.d.ts +16 -2
  5. package/dist/cancel.js +36 -2
  6. package/dist/channels.js +20 -6
  7. package/dist/config.d.ts +49 -1
  8. package/dist/config.js +86 -7
  9. package/dist/connection.d.ts +52 -10
  10. package/dist/connection.js +305 -73
  11. package/dist/handles.d.ts +10 -1
  12. package/dist/handles.js +27 -4
  13. package/dist/index.d.ts +4 -4
  14. package/dist/jobs.d.ts +33 -1
  15. package/dist/jobs.js +45 -0
  16. package/dist/logger.d.ts +22 -10
  17. package/dist/logger.js +118 -28
  18. package/dist/metrics.d.ts +19 -0
  19. package/dist/metrics.js +22 -1
  20. package/dist/query.js +33 -9
  21. package/dist/records.d.ts +30 -2
  22. package/dist/records.js +8 -0
  23. package/dist/runtime/addon.js +3 -2
  24. package/dist/runtime/bun.js +33 -12
  25. package/dist/runtime/node.js +30 -8
  26. package/dist/runtime/pipe.d.ts +2 -0
  27. package/dist/runtime/pipe.js +45 -20
  28. package/dist/runtime/shared.d.ts +28 -4
  29. package/dist/runtime/shared.js +125 -13
  30. package/dist/runtime.d.ts +28 -6
  31. package/dist/session.d.ts +9 -5
  32. package/dist/session.js +113 -31
  33. package/dist/sql.d.ts +18 -5
  34. package/dist/sql.js +44 -11
  35. package/dist/store.d.ts +25 -11
  36. package/dist/store.js +67 -9
  37. package/dist/wire/codec.d.ts +5 -0
  38. package/dist/wire/codec.js +10 -0
  39. package/dist/wire/frame.d.ts +2 -0
  40. package/dist/wire/frame.js +4 -0
  41. package/dist/wire/msgpack.d.ts +7 -3
  42. package/dist/wire/msgpack.js +11 -4
  43. package/dist/wire/protocol.d.ts +246 -36
  44. package/dist/wire/protocol.js +387 -101
  45. package/package.json +9 -8
  46. package/src/blobs.ts +13 -5
  47. package/src/cancel.ts +44 -2
  48. package/src/channels.ts +21 -6
  49. package/src/config.ts +138 -12
  50. package/src/connection.ts +377 -77
  51. package/src/handles.ts +32 -4
  52. package/src/index.ts +5 -0
  53. package/src/jobs.ts +63 -0
  54. package/src/logger.ts +158 -40
  55. package/src/metrics.ts +46 -1
  56. package/src/query.ts +33 -13
  57. package/src/records.ts +40 -2
  58. package/src/runtime/addon.ts +3 -2
  59. package/src/runtime/bun.ts +44 -9
  60. package/src/runtime/node.ts +30 -5
  61. package/src/runtime/pipe.ts +49 -19
  62. package/src/runtime/shared.ts +137 -16
  63. package/src/runtime.ts +31 -6
  64. package/src/session.ts +129 -30
  65. package/src/sql.ts +48 -14
  66. package/src/store.ts +81 -14
  67. package/src/wire/codec.ts +11 -0
  68. package/src/wire/frame.ts +5 -0
  69. package/src/wire/msgpack.ts +11 -4
  70. package/src/wire/protocol.ts +404 -103
package/dist/handles.js CHANGED
@@ -1,5 +1,6 @@
1
1
  import { goesAgain } from './connection.js';
2
2
  import { InvalidError } from './errors.js';
3
+ import { ConnectionLostError } from './session.js';
3
4
  import { Reader, Writer } from './wire/msgpack.js';
4
5
  import { Handle } from './wire/protocol.js';
5
6
  /** Asks through `call`, the body and the answer each in bytes of their own. */
@@ -84,7 +85,9 @@ export function handleOn(connection, method, open) {
84
85
  const key = `${method}:${encoded}`;
85
86
  let opening = connection.handles.get(key);
86
87
  if (opening === undefined) {
87
- opening = connection.session.call(method, open).then(body => Handle.decode(body).handle ?? 0);
88
+ opening = connection.session.call(method, open).then(body => Handle.decode(body).handle ?? 0, (err) => {
89
+ throw unsent(err);
90
+ });
88
91
  connection.handles.set(key, opening);
89
92
  opening.catch(() => connection.handles.delete(key));
90
93
  }
@@ -94,10 +97,30 @@ export function handleOn(connection, method, open) {
94
97
  }
95
98
  return opening;
96
99
  }
97
- /** A bucket's or a queue's name, as every engine takes it: [a-z0-9][a-z0-9_-]{0,63}. */
100
+ /**
101
+ * A connection lost while a handle opened, as the call that needed the handle
102
+ * meets it: the open's REQUEST had left, the call's own had not, so the call
103
+ * goes again on the next connection, where the handle opens again.
104
+ */
105
+ export function unsent(err) {
106
+ return err instanceof ConnectionLostError && err.sent
107
+ ? new ConnectionLostError(err.message, false)
108
+ : err;
109
+ }
110
+ const named = /^[a-z0-9][a-z0-9_-]{0,63}$/;
111
+ /**
112
+ * A bucket's or a queue's name, as every engine takes it: [a-z0-9][a-z0-9_-]{0,63},
113
+ * with the nearest name that is when there is one: `focusroom.rooms` as `focusroom_rooms`.
114
+ */
98
115
  export function checkName(name, of) {
99
- if (!/^[a-z0-9][a-z0-9_-]{0,63}$/.test(name)) {
100
- throw new InvalidError(`the ${of} name ${JSON.stringify(name)}: a name is [a-z0-9][a-z0-9_-]{0,63}`);
116
+ if (!named.test(name)) {
117
+ const nearest = name
118
+ .toLowerCase()
119
+ .replace(/[^a-z0-9_-]/g, '_')
120
+ .replace(/^[_-]+/, '')
121
+ .slice(0, 64);
122
+ const offered = named.test(nearest) ? `, as ${JSON.stringify(nearest)}` : '';
123
+ throw new InvalidError(`the ${of} name ${JSON.stringify(name)}: a name is [a-z0-9][a-z0-9_-]{0,63}${offered}`);
101
124
  }
102
125
  }
103
126
  /** Answers what a batch's function returned, once the batch's calls have settled. */
package/dist/index.d.ts CHANGED
@@ -3,20 +3,20 @@ export type { Body, FileBytes, FileInfo, FileListOptions, FileOptions, FilePage,
3
3
  export { withSignal } from './cancel.ts';
4
4
  export type { Channel, EachOptions, Message as ChannelMessage, Missed, SubscribeOptions, Subscription, TxChannel, } from './channels.ts';
5
5
  export type { Clock } from './clock.ts';
6
- export { type Bounds, type Changes, type Config, type ConfigOptions, type DurationBounds, duration, type EnvLayer, type EnvOptions, type Explained, type ExplainLine, env, type FileLayer, file, int, type Layer, list, live, type Overlay, oneOf, required, Secret, type Setting, secret, type UpdateOptions, type Value, type Vars, } from './config.ts';
6
+ export { type Bounds, type Changes, type Config, type ConfigOptions, type DurationBounds, duration, type EnvLayer, type EnvOptions, type Explained, type ExplainGiven, type ExplainLine, type ExplainOrigin, env, type FileLayer, file, int, type Layer, list, live, type Overlay, oneOf, required, Secret, type Setting, secret, type UpdateOptions, type Value, type Vars, } from './config.ts';
7
7
  export type { Durability, SelfMetrics } from './connection.ts';
8
8
  export { type ConsoleFormat, type ConsoleTime, secrets } from './console.ts';
9
9
  export * from './errors.ts';
10
10
  export type { Page } from './handles.ts';
11
- export type { AddOptions, Answer, Backoff, Concurrency, Handler, Job, JobOptions, JobPage, JobState, ListOptions, Queue, QueueOptions, Run, Schedule, ScheduleOptions, ScheduleWhen, StoreTxQueue, TxQueue, Worker, WorkOptions, } from './jobs.ts';
11
+ export type { Adding, AddOptions, Answer, Backoff, Concurrency, Handler, Job, JobOptions, JobPage, JobState, ListOptions, Queue, QueueOptions, Run, Schedule, ScheduleOptions, ScheduleWhen, StoreTxQueue, TxQueue, Worker, WorkOptions, } from './jobs.ts';
12
12
  export type { Bucket, BucketOptions, Counters, CountersOptions, Entry, Page as KvPage, ValueType, WriteOptions, } from './kv.ts';
13
13
  export type { Allowance, Quota, Rate, RateLimit, RateLimitOptions, WindowUse } from './limits.ts';
14
14
  export type { HoldOptions, Lease, Lock, LockOptions, Permit, Semaphore, SemaphoreOptions, Tried, } from './locks.ts';
15
15
  export { type ConsoleOut, type Logger, type LoggerOptions, logger } from './logger.ts';
16
- export { type Aggregate, type AggregateQuery, type Bucket as MetricsBucket, type Buckets, type Condition as MetricsCondition, type Counter, type Damage as MetricsDamage, type Description as MetricsDescription, exponentialBuckets, type Gauge, type GaugeFuncOptions, type Histogram, type HistogramOptions, type InstrumentLabels, type InstrumentOptions, type LabelPage, type Labels, type LabelValue, type Metrics, type NameInfo, type NamePage, type Newest, type Op as MetricsOp, type PageOptions, type Percentile, type PercentileBucket, type Plan, type Query as MetricsQuery, type Series, type SeriesInput, type SeriesKind, type Timer, type TimerOptions, type Timing, type ValueBucket, type ValuePage, } from './metrics.ts';
16
+ export { type Aggregate, type AggregateQuery, type Bucket as MetricsBucket, type Buckets, type Condition as MetricsCondition, type Counter, type Damage as MetricsDamage, type Description as MetricsDescription, exponentialBuckets, type Gauge, type GaugeFuncOptions, type Histogram, type HistogramOptions, type InstrumentLabels, type InstrumentOptions, type LabelPage, type Labels, type LabelValue, type Metrics, type NameInfo, type NamePage, type Newest, type Op as MetricsOp, type PageOptions, type Percentile, type PercentileBucket, type Plan, type PrometheusOptions, type Query as MetricsQuery, type Series, type SeriesInput, type SeriesKind, type Timer, type TimerOptions, type Timing, type ValueBucket, type ValuePage, } from './metrics.ts';
17
17
  export type { Once, OnceOptions } from './once.ts';
18
18
  export type { ColumnType, Query, RowPage, Table, TableOptions, UpsertOptions } from './query.ts';
19
- export type { Aggregate as EventsAggregate, AllEvents, AppendOptions, Counted, Damage, Event, EventLevel, Events, EventsOptions, Fields, Level, Lines, NewEvent, Page as EventsPage, PollBatch, Query as EventsQuery, Row as EventsRow, Stats, StoreLoggerOptions, StreamInfo, TailBatch, TailQuery, Top, } from './records.ts';
19
+ export type { Aggregate as EventsAggregate, AllEvents, AppendOptions, Counted, Damage, Event, EventLevel, Events, EventsOptions, Fields, Level, Levels, Lines, NewEvent, Page as EventsPage, PollBatch, Query as EventsQuery, Row as EventsRow, Stats, StoreLoggerOptions, StreamInfo, TailBatch, TailQuery, Top, } from './records.ts';
20
20
  export type { StandardSchemaV1 } from './schema.ts';
21
21
  export { type Found as SearchFound, type Hit as SearchHit, type IndexDocument, type IndexInfo, type IndexOptions, type IndexStatus, type Query as SearchQuery, SearchIndex, search, TableIndex, type TableIndexOptions, type Vector, type Where, type WhereValue, } from './search.ts';
22
22
  export { type Columns, type Database, type DatabaseOptions, type Done, type Migrations, type Row, Sql, type SqlArg, type SqlTag, type SqlTx, type SqlValue, type Statement, sql, } from './sql.ts';
package/dist/jobs.d.ts CHANGED
@@ -1,4 +1,4 @@
1
- import type { Link } from './connection.ts';
1
+ import type { Connection, Link } from './connection.ts';
2
2
  import { type Home, type Open, type Via } from './handles.ts';
3
3
  import { type Rate } from './limits.ts';
4
4
  import { type StandardSchemaV1 } from './schema.ts';
@@ -239,6 +239,17 @@ export declare class Queue<T> {
239
239
  * await reminders.add({ userId: 42, text: 'Call mom' }, { delay: '1h' })
240
240
  */
241
241
  add(value: T, options?: AddOptions): Promise<boolean>;
242
+ /**
243
+ * A job's add for a batch of the database the queue was opened from:
244
+ * `db.batch` writes it with the batch's statements, so the job commits
245
+ * with the rows or not at all, and shares the commit with every other
246
+ * write of the file, where `db.tx` holds the writer and pays a sync of
247
+ * its own. It adds as `add` does, unless its id is taken: its `Done` in
248
+ * the batch's answer changed 1 job, or 0.
249
+ *
250
+ * await db.batch([sql`update notes set body = ${body} where id = ${id}`, indexing.adding({ noteId: id })])
251
+ */
252
+ adding(value: T, options?: AddOptions): Adding;
242
253
  /**
243
254
  * Makes the id's job this value at this time, whatever it was: one waiting
244
255
  * is made new, a failed one starts again, and a running one runs once more
@@ -314,6 +325,27 @@ export declare class TxQueue<T> {
314
325
  /** The job under id as the transaction sees it, its own writes among it. */
315
326
  get(id: string): Promise<Job<T> | undefined>;
316
327
  }
328
+ /** A queue's add as a database's batch carries it: the queue, and the add's call but its handle. */
329
+ interface AddingParts {
330
+ name: string;
331
+ source: Source<unknown>;
332
+ call: Omit<Written<typeof JobsCall.fields>, 'handle'>;
333
+ }
334
+ /** A job's add that a database's batch carries, from `queue.adding`: it commits with the batch's statements, or not at all. */
335
+ export declare class Adding {
336
+ #private;
337
+ /** Comes from `queue.adding`. */
338
+ constructor(parts: AddingParts);
339
+ }
340
+ /**
341
+ * What a batch of `home` sends of an add, on the connection it goes on: the
342
+ * add's call, its queue's handle opened there first. A queue kept outside
343
+ * `home` is refused, since its job would not commit with the rows.
344
+ */
345
+ export declare function addingIn(adding: Adding, home: Home): (connection: Connection) => Promise<{
346
+ method: number;
347
+ body: Uint8Array;
348
+ }>;
317
349
  /** A write the store's transaction keeps for its commit: its method, and its call or the id it cancels. */
318
350
  export interface JobOp {
319
351
  source: Source<unknown>;
package/dist/jobs.js CHANGED
@@ -288,6 +288,7 @@ async function* watchJob(source, id) {
288
288
  let last;
289
289
  for (;;) {
290
290
  const stream = await source.link.run('read', connection => openWatch(connection, source, id));
291
+ const unfollow = source.link.follow(stream);
291
292
  try {
292
293
  for (;;) {
293
294
  const event = await stream.next();
@@ -309,6 +310,7 @@ async function* watchJob(source, id) {
309
310
  }
310
311
  finally {
311
312
  stream.cancel();
313
+ unfollow();
312
314
  }
313
315
  }
314
316
  }
@@ -356,6 +358,20 @@ export class Queue {
356
358
  };
357
359
  return (await this.#changed(methods['jobs.add'], fields)) === true;
358
360
  }
361
+ /**
362
+ * A job's add for a batch of the database the queue was opened from:
363
+ * `db.batch` writes it with the batch's statements, so the job commits
364
+ * with the rows or not at all, and shares the commit with every other
365
+ * write of the file, where `db.tx` holds the writer and pays a sync of
366
+ * its own. It adds as `add` does, unless its id is taken: its `Done` in
367
+ * the batch's answer changed 1 job, or 0.
368
+ *
369
+ * await db.batch([sql`update notes set body = ${body} where id = ${id}`, indexing.adding({ noteId: id })])
370
+ */
371
+ adding(value, options = {}) {
372
+ const call = { ...jobFields(options), id: options.id, value: this.#source.values.encode(value) };
373
+ return new Adding({ name: this.name, source: this.#source, call });
374
+ }
359
375
  /**
360
376
  * Makes the id's job this value at this time, whatever it was: one waiting
361
377
  * is made new, a failed one starts again, and a running one runs once more
@@ -483,6 +499,35 @@ export class TxQueue {
483
499
  return this.#queue.get(id);
484
500
  }
485
501
  }
502
+ // reads an add's parts, which only this module sees
503
+ let partsOf;
504
+ /** A job's add that a database's batch carries, from `queue.adding`: it commits with the batch's statements, or not at all. */
505
+ export class Adding {
506
+ #parts;
507
+ /** Comes from `queue.adding`. */
508
+ constructor(parts) {
509
+ this.#parts = parts;
510
+ }
511
+ static {
512
+ partsOf = adding => adding.#parts;
513
+ }
514
+ }
515
+ /**
516
+ * What a batch of `home` sends of an add, on the connection it goes on: the
517
+ * add's call, its queue's handle opened there first. A queue kept outside
518
+ * `home` is refused, since its job would not commit with the rows.
519
+ */
520
+ export function addingIn(adding, home) {
521
+ const { name, source, call } = partsOf(adding);
522
+ if (source.home !== home) {
523
+ const kept = source.home === undefined ? 'jobs.db' : source.home.describe;
524
+ throw new InvalidError(`${home.describe}: jobs queue ${name}: kept in ${kept}, outside this batch of ${home.describe}`);
525
+ }
526
+ return async (connection) => {
527
+ const handle = await handleOn(connection, source.openMethod, source.open);
528
+ return { method: methods['jobs.add'], body: JobsCall.encode({ ...call, handle }) };
529
+ };
530
+ }
486
531
  /**
487
532
  * A queue's writes inside the store's transaction, from `tx.with(queue)` in
488
533
  * `store.tx`: each is kept for the commit, which makes them all or none, so
package/dist/logger.d.ts CHANGED
@@ -1,11 +1,11 @@
1
1
  import { DropNotice, FailureLog, type Say } from './background.ts';
2
2
  import { type EnvLayer, type Setting } from './config.ts';
3
3
  import { type ConsoleFormat, type ConsoleLine, type ConsoleTime, type Redactor } from './console.ts';
4
- import type { Fields, Level } from './records.ts';
4
+ import type { Fields, Level, Levels } from './records.ts';
5
5
  export interface LoggerOptions {
6
6
  /** the lines held between writes; past it a line is dropped and counted: 1024 */
7
7
  buffer?: number;
8
- /** the least level a line is kept at; every level when absent */
8
+ /** the least level a line is printed at, every level when absent */
9
9
  level?: Level;
10
10
  /** how a line is written as it is logged: pretty on a terminal and JSON otherwise when absent */
11
11
  console?: ConsoleFormat;
@@ -24,12 +24,16 @@ export interface LoggerOptions {
24
24
  redact?: readonly string[];
25
25
  /** leaves a URL's password in a value as it is, which is otherwise hidden */
26
26
  keepUrlPasswords?: boolean;
27
- /** changes each field's value before it is hidden, kept and shown */
27
+ /**
28
+ * changes each field's value before it is hidden, kept and shown, and
29
+ * undefined leaves the field out: return the value it was given for a
30
+ * field it leaves as it is
31
+ */
28
32
  replace?: (key: string, value: unknown) => unknown;
29
33
  /**
30
- * where LOG_LEVEL, LOG_FORMAT and LOG_TIME are read, which win over the
31
- * options: `env('APP')` reads APP_LOG_LEVEL and the rest, and false
32
- * reads nothing; the bare names when absent
34
+ * where LOG_LEVEL, LOG_CONSOLE_LEVEL, LOG_FORMAT and LOG_TIME are read,
35
+ * each of which wins over its option: `env('APP')` reads APP_LOG_LEVEL
36
+ * and the rest, and false reads nothing; the bare names when absent
33
37
  */
34
38
  env?: EnvLayer | false;
35
39
  /** adds where each line was logged: source={"function":…,"file":…,"line":…} */
@@ -46,7 +50,10 @@ interface Shared {
46
50
  append: ((lines: ConsoleLine[]) => Promise<void>) | undefined;
47
51
  stream: string;
48
52
  most: number;
49
- least: number;
53
+ /** the least level kept, which the config log changes while the logger runs */
54
+ kept: number;
55
+ /** the least level printed, LOG_CONSOLE_LEVEL's or level's console part; undefined prints what is kept */
56
+ printed: number | undefined;
50
57
  redact: Redactor | undefined;
51
58
  replace: ((key: string, value: unknown) => unknown) | undefined;
52
59
  source: boolean;
@@ -88,8 +95,12 @@ type LogDeclaration = {
88
95
  format: Setting<string>;
89
96
  time: Setting<string>;
90
97
  };
98
+ /** What a logger is told: a logger's options, a store's logger's level in its two parts. */
99
+ type Told = Omit<LoggerOptions, 'level'> & {
100
+ level?: Level | Levels | undefined;
101
+ };
91
102
  /** A store's logger of one stream, and what it declares its config `log` with. */
92
- export declare function newStoreLogger(append: (lines: ConsoleLine[]) => Promise<void>, stream: string, options?: LoggerOptions): [Logger, LogConfig];
103
+ export declare function newStoreLogger(append: (lines: ConsoleLine[]) => Promise<void>, stream: string, options?: Told): [Logger, LogConfig];
93
104
  /** TinyStore's own lines, on the console of a logger of stream tinystore made when first needed. */
94
105
  export declare const sayOwn: Say;
95
106
  /**
@@ -129,8 +140,9 @@ export declare class Logger {
129
140
  }
130
141
  /**
131
142
  * Fields as key, JSON pairs, as a console writes them and the store keeps
132
- * them: an Error's value its stack, which JSON would write as {}, a secret's
133
- * value hidden at any depth, and a URL's password unless kept.
143
+ * them: a field whose value is undefined left out, as JSON.stringify leaves
144
+ * it out of an object, an Error's value its stack, which JSON would write as
145
+ * {}, a secret's value hidden at any depth, and a URL's password unless kept.
134
146
  */
135
147
  export declare function encodeFields(fields: Fields, redact: Redactor | undefined, replace?: (key: string, value: unknown) => unknown): string[];
136
148
  export {};
package/dist/logger.js CHANGED
@@ -35,10 +35,26 @@ export function logger(stream, options) {
35
35
  export function newLogger(append, stream, options = {}, out) {
36
36
  return new Logger(sharedOf(append, stream, options, out).shared, []);
37
37
  }
38
+ /**
39
+ * A level's two parts, the least kept and the least printed: one level is
40
+ * the kept alone, which the console then follows.
41
+ */
42
+ function partsOf(level) {
43
+ if (typeof level !== 'object') {
44
+ return [level, undefined];
45
+ }
46
+ for (const part of Object.keys(level)) {
47
+ if (part !== 'store' && part !== 'console') {
48
+ throw new InvalidError(`a logger's level of ${JSON.stringify(part)}: its parts are store and console`);
49
+ }
50
+ }
51
+ return [level.store, level.console];
52
+ }
38
53
  /** A store's logger of one stream, and what it declares its config `log` with. */
39
54
  export function newStoreLogger(append, stream, options = {}) {
40
55
  const { shared, spelled } = sharedOf(append, stream, options, undefined);
41
- const level = options.level === undefined ? 'trace' : nameOf(levelOf(options.level));
56
+ const [kept] = partsOf(options.level);
57
+ const level = kept === undefined ? 'trace' : nameOf(levelOf(kept));
42
58
  const format = options.console ?? (prettyUntold(options, undefined) ? 'pretty' : 'json');
43
59
  const time = options.time ?? 'clock';
44
60
  const others = (all, one) => all.filter(name => name !== one);
@@ -48,9 +64,10 @@ export function newStoreLogger(append, stream, options = {}) {
48
64
  time: oneOf(time, others(times, time)),
49
65
  };
50
66
  const env = options.env ?? environmentOf('');
67
+ // the level kept, and printed too while the level has no console part
51
68
  const take = (name) => {
52
69
  if (Object.hasOwn(levels, name)) {
53
- shared.least = levels[name];
70
+ shared.kept = levels[name];
54
71
  }
55
72
  };
56
73
  if (env === false) {
@@ -82,7 +99,8 @@ function sharedOf(append, stream, options, out) {
82
99
  append,
83
100
  stream,
84
101
  most,
85
- least: read.level === undefined ? Number.NEGATIVE_INFINITY : levelOf(read.level),
102
+ kept: read.kept === undefined ? Number.NEGATIVE_INFINITY : levelOf(read.kept),
103
+ printed: read.printed === undefined ? undefined : levelOf(read.printed),
86
104
  redact: redactor(read.redact, read.keepUrlPasswords),
87
105
  replace: read.replace,
88
106
  source: read.source === true,
@@ -98,12 +116,16 @@ function sharedOf(append, stream, options, out) {
98
116
  return { shared, spelled };
99
117
  }
100
118
  /**
101
- * LOG_LEVEL, LOG_FORMAT and LOG_TIME over the options, after the prefix env
102
- * names, as Go and Python read them; a console turned off stays off. Each
103
- * variable read is spelled as the config `log` reads it, by its name.
119
+ * LOG_LEVEL, LOG_CONSOLE_LEVEL, LOG_FORMAT and LOG_TIME over the options,
120
+ * each over its own, after the prefix env names, as Go and Python read them;
121
+ * a console turned off stays off. Each variable read is spelled as the
122
+ * config `log` reads it, by its name, but LOG_CONSOLE_LEVEL, which no
123
+ * setting of the config holds.
104
124
  */
105
125
  function fromEnvironment(options) {
106
- const read = { ...options };
126
+ const { level: given, ...rest } = options;
127
+ const [kept, printed] = partsOf(given);
128
+ const read = { ...rest, kept, printed };
107
129
  const ignored = [];
108
130
  const spelled = {};
109
131
  const env = options.env ?? environmentOf('');
@@ -112,16 +134,14 @@ function fromEnvironment(options) {
112
134
  }
113
135
  const variables = environment(env);
114
136
  const name = (variable) => (env.prefix === '' ? variable : `${env.prefix}_${variable}`);
115
- const level = variables(name('LOG_LEVEL'));
137
+ const level = levelIn(variables, name('LOG_LEVEL'), ignored);
116
138
  if (level !== undefined) {
117
- const known = level.toLowerCase() === 'warning' ? 'warn' : level.toLowerCase();
118
- if (Object.hasOwn(levels, known)) {
119
- read.level = levels[known];
120
- spelled[name('LOG_LEVEL')] = known;
121
- }
122
- else {
123
- ignored.push([name('LOG_LEVEL'), level, 'trace, debug, info, warn or error']);
124
- }
139
+ read.kept = levels[level];
140
+ spelled[name('LOG_LEVEL')] = level;
141
+ }
142
+ const shown = levelIn(variables, name('LOG_CONSOLE_LEVEL'), ignored);
143
+ if (shown !== undefined) {
144
+ read.printed = levels[shown];
125
145
  }
126
146
  const format = variables(name('LOG_FORMAT'));
127
147
  if (format !== undefined) {
@@ -147,6 +167,19 @@ function fromEnvironment(options) {
147
167
  }
148
168
  return [read, ignored, spelled];
149
169
  }
170
+ /** The level a variable names, the case ignored and warning read as warn; a value it cannot mean is ignored. */
171
+ function levelIn(variables, name, ignored) {
172
+ const text = variables(name);
173
+ if (text === undefined) {
174
+ return undefined;
175
+ }
176
+ const known = text.toLowerCase() === 'warning' ? 'warn' : text.toLowerCase();
177
+ if (Object.hasOwn(levels, known)) {
178
+ return known;
179
+ }
180
+ ignored.push([name, text, 'trace, debug, info, warn or error']);
181
+ return undefined;
182
+ }
150
183
  /** The variables of env's files, the process's or `from`'s over them, a missing file skipped; an empty one is none. */
151
184
  function environment(env) {
152
185
  const fromFiles = {};
@@ -301,11 +334,13 @@ export class Logger {
301
334
  }
302
335
  }
303
336
  #log(level, message, attrs) {
304
- if (level < this.#shared.least) {
337
+ // a line that no place of the logger takes is never encoded
338
+ if (!prints(this.#shared, level) && !keeps(this.#shared, level)) {
305
339
  return;
306
340
  }
307
341
  this.#emit({ level, msg: message }, attrs);
308
342
  }
343
+ /** Prints a line from the level printed and keeps it from the level kept; an event, which has none, both. */
309
344
  #emit(what, attrs) {
310
345
  const trace = traceOf(currentTrace());
311
346
  const shared = this.#shared;
@@ -318,8 +353,12 @@ export class Logger {
318
353
  attrs: shared.source ? [...encoded, sourceKey, sourceOfCaller()] : encoded,
319
354
  ...trace,
320
355
  };
321
- this.#shared.print?.(line);
322
- this.#queue(line);
356
+ if (what.level === undefined || prints(shared, what.level)) {
357
+ shared.print?.(line);
358
+ }
359
+ if (what.level === undefined || keeps(shared, what.level)) {
360
+ this.#queue(line);
361
+ }
323
362
  }
324
363
  #queue(line) {
325
364
  const shared = this.#shared;
@@ -341,6 +380,14 @@ export class Logger {
341
380
  }
342
381
  }
343
382
  }
383
+ /** Whether a logger prints a line of level: it has a console, and the level is the level printed or above. */
384
+ function prints(shared, level) {
385
+ return shared.print !== undefined && level >= (shared.printed ?? shared.kept);
386
+ }
387
+ /** Whether a logger keeps a line of level: it is a store's, and the level is its level or above. */
388
+ function keeps(shared, level) {
389
+ return shared.append !== undefined && level >= shared.kept;
390
+ }
344
391
  /**
345
392
  * Writes what the buffer holds, and again while half of it filled during the
346
393
  * write before. A full buffer drops a line before the line could ask for a
@@ -350,21 +397,60 @@ export class Logger {
350
397
  async function writeWaiting(shared, append) {
351
398
  let batch = shared.waiting.splice(0);
352
399
  while (batch.length > 0) {
353
- try {
354
- await append(batch);
355
- shared.failures?.succeeded();
356
- }
357
- catch (err) {
358
- shared.dropped += batch.length;
359
- shared.failures?.failed(err);
400
+ for (const piece of pieces(batch)) {
401
+ try {
402
+ await append(piece);
403
+ shared.failures?.succeeded();
404
+ }
405
+ catch (err) {
406
+ shared.dropped += piece.length;
407
+ shared.failures?.failed(err);
408
+ }
360
409
  }
361
410
  batch = shared.waiting.length >= shared.most / 2 ? shared.waiting.splice(0) : [];
362
411
  }
363
412
  }
413
+ /**
414
+ * The text a write carries at most. A request's body holds 2 MiB, and a
415
+ * buffer of long lines, or one made larger, held more: the whole write was
416
+ * refused and every line in it dropped. Counted in UTF-16 units, three bytes
417
+ * each at worst, so a piece stays within the body however its text is spelt.
418
+ */
419
+ const PIECE = 512 * 1024;
420
+ /** The lines in writes of at most PIECE of text each, in their order. */
421
+ function pieces(lines) {
422
+ const out = [];
423
+ let piece = [];
424
+ let size = 0;
425
+ for (const line of lines) {
426
+ const weight = textOf(line);
427
+ if (piece.length > 0 && size + weight > PIECE) {
428
+ out.push(piece);
429
+ piece = [];
430
+ size = 0;
431
+ }
432
+ piece.push(line);
433
+ size += weight;
434
+ }
435
+ if (piece.length > 0) {
436
+ out.push(piece);
437
+ }
438
+ return out;
439
+ }
440
+ /** A line's text, with a little for its time, level and framing. */
441
+ function textOf(line) {
442
+ let size = 64 + (line.msg?.length ?? 0) + (line.event?.length ?? 0);
443
+ for (const part of line.attrs)
444
+ size += part.length;
445
+ for (const part of line.context)
446
+ size += part.length;
447
+ return size;
448
+ }
364
449
  /**
365
450
  * Fields as key, JSON pairs, as a console writes them and the store keeps
366
- * them: an Error's value its stack, which JSON would write as {}, a secret's
367
- * value hidden at any depth, and a URL's password unless kept.
451
+ * them: a field whose value is undefined left out, as JSON.stringify leaves
452
+ * it out of an object, an Error's value its stack, which JSON would write as
453
+ * {}, a secret's value hidden at any depth, and a URL's password unless kept.
368
454
  */
369
455
  export function encodeFields(fields, redact, replace) {
370
456
  const pairs = Array.isArray(fields)
@@ -373,6 +459,10 @@ export function encodeFields(fields, redact, replace) {
373
459
  const flat = [];
374
460
  for (const [key, given] of pairs) {
375
461
  const value = replace === undefined ? given : replace(key, given);
462
+ // before it is hidden, as Rust's replace drops a field: a secret given no value is no field
463
+ if (value === undefined) {
464
+ continue;
465
+ }
376
466
  const shown = value instanceof Error ? (value.stack ?? String(value)) : value;
377
467
  if (redact?.hides(key)) {
378
468
  flat.push(key, redacted);
package/dist/metrics.d.ts CHANGED
@@ -42,6 +42,15 @@ export interface SeriesInput {
42
42
  /** how long the name's samples stay: its own, kept in the file; unless given, what it keeps already */
43
43
  keep?: Duration | undefined;
44
44
  }
45
+ /** What an ingest of an exporter's text gives every series of it. */
46
+ export interface PrometheusOptions {
47
+ /** labels every series gets beside its own, such as the instance scraped; one the text gives too is refused, naming its line */
48
+ labels?: Record<string, LabelValue> | undefined;
49
+ /** the time of every sample the text gives none; the store's clock unless given */
50
+ at?: Time | undefined;
51
+ /** how long every name of the text keeps its samples */
52
+ keep?: Duration | undefined;
53
+ }
45
54
  /** A series as a read gives it: its samples as two columns. */
46
55
  export interface Series {
47
56
  name: string;
@@ -118,6 +127,7 @@ export interface Plan {
118
127
  bytes: number;
119
128
  decoded: number;
120
129
  exceeds: {
130
+ /** series, bytes read, samples decoded or answer */
121
131
  limit: string;
122
132
  wanted: number;
123
133
  bound: number;
@@ -177,6 +187,15 @@ export declare class Metrics {
177
187
  constructor(link: Link, instruments: Instruments);
178
188
  /** Writes series and their samples, every one or none: a refused series names itself. */
179
189
  ingest(series: SeriesInput | readonly SeriesInput[]): Promise<void>;
190
+ /**
191
+ * Reads the text an exporter serves, in Prometheus' format or OpenMetrics', and writes its
192
+ * series, every one or none: a counter as a counter, a histogram's _bucket series as
193
+ * histogram. A line it cannot read is refused, naming the line.
194
+ *
195
+ * const text = await (await fetch('http://web-1:9100/metrics')).text()
196
+ * await store.metrics.ingestPrometheus(text, { labels: { instance: 'web-1' } })
197
+ */
198
+ ingestPrometheus(text: string | Uint8Array, options?: PrometheusOptions): Promise<void>;
180
199
  /** The samples of each series the query finds, in its range: whole, or a limit error naming its bound. */
181
200
  read(query: Query): Promise<Series[]>;
182
201
  /** The newest sample of each series the query finds, when it has one in the range. */
package/dist/metrics.js CHANGED
@@ -8,7 +8,7 @@ import { download } from './connection.js';
8
8
  import { InvalidError } from './errors.js';
9
9
  import { Counter, Gauge, Histogram, labelsOf, Timer, } from './instruments.js';
10
10
  import { ms, unixMs } from './time.js';
11
- import { Empty, MetricsAggregateAsked, MetricsAggregates, MetricsDamages, MetricsDeleted, MetricsDescription, MetricsIngest, MetricsLatest, MetricsNameAsked, MetricsNamePage, MetricsPageAsked, MetricsPlan, MetricsQuery, MetricsRead, MetricsSeriesNamed, MetricsTextPage, methods, } from './wire/protocol.js';
11
+ import { Empty, MetricsAggregateAsked, MetricsAggregates, MetricsDamages, MetricsDeleted, MetricsDescription, MetricsExposition, MetricsIngest, MetricsLatest, MetricsNameAsked, MetricsNamePage, MetricsPageAsked, MetricsPlan, MetricsQuery, MetricsRead, MetricsSeriesNamed, MetricsTextPage, methods, } from './wire/protocol.js';
12
12
  export { Counter, exponentialBuckets, Gauge, Histogram, Timer, Timing, } from './instruments.js';
13
13
  /**
14
14
  * A store's metrics: series by name and labels, their reads and aggregates,
@@ -30,6 +30,27 @@ export class Metrics {
30
30
  const body = MetricsIngest.encode({ series: given.map(newSeries) });
31
31
  await this.#link.run('write', connection => connection.session.call(methods['metrics.ingest'], body));
32
32
  }
33
+ /**
34
+ * Reads the text an exporter serves, in Prometheus' format or OpenMetrics', and writes its
35
+ * series, every one or none: a counter as a counter, a histogram's _bucket series as
36
+ * histogram. A line it cannot read is refused, naming the line.
37
+ *
38
+ * const text = await (await fetch('http://web-1:9100/metrics')).text()
39
+ * await store.metrics.ingestPrometheus(text, { labels: { instance: 'web-1' } })
40
+ */
41
+ async ingestPrometheus(text, options = {}) {
42
+ const labels = {};
43
+ for (const [label, value] of Object.entries(options.labels ?? {})) {
44
+ labels[label] = String(value);
45
+ }
46
+ const body = MetricsExposition.encode({
47
+ text: typeof text === 'string' ? new TextEncoder().encode(text) : text,
48
+ labels,
49
+ at: options.at === undefined ? undefined : unixMs(options.at),
50
+ keep: options.keep === undefined ? undefined : ms(options.keep),
51
+ });
52
+ await this.#link.run('write', connection => connection.session.call(methods['metrics.ingestPrometheus'], body));
53
+ }
33
54
  /** The samples of each series the query finds, in its range: whole, or a limit error naming its bound. */
34
55
  async read(query) {
35
56
  const parts = await this.#download('metrics.read', MetricsQuery.encode(queryOf(query)));