@tinyshed/tinystore 0.1.0-rc.7 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/blobs.d.ts +1 -1
- package/dist/blobs.js +13 -5
- package/dist/cancel.d.ts +16 -2
- package/dist/cancel.js +36 -2
- package/dist/channels.js +20 -6
- package/dist/config.d.ts +49 -1
- package/dist/config.js +86 -7
- package/dist/connection.d.ts +52 -10
- package/dist/connection.js +305 -73
- package/dist/handles.d.ts +10 -1
- package/dist/handles.js +27 -4
- package/dist/index.d.ts +4 -4
- package/dist/jobs.d.ts +33 -1
- package/dist/jobs.js +45 -0
- package/dist/logger.d.ts +22 -10
- package/dist/logger.js +118 -28
- package/dist/metrics.d.ts +19 -0
- package/dist/metrics.js +22 -1
- package/dist/query.js +33 -9
- package/dist/records.d.ts +30 -2
- package/dist/records.js +8 -0
- package/dist/runtime/addon.js +3 -2
- package/dist/runtime/bun.js +33 -12
- package/dist/runtime/node.js +30 -8
- package/dist/runtime/pipe.d.ts +2 -0
- package/dist/runtime/pipe.js +45 -20
- package/dist/runtime/shared.d.ts +28 -4
- package/dist/runtime/shared.js +125 -13
- package/dist/runtime.d.ts +28 -6
- package/dist/session.d.ts +9 -5
- package/dist/session.js +113 -31
- package/dist/sql.d.ts +18 -5
- package/dist/sql.js +44 -11
- package/dist/store.d.ts +25 -11
- package/dist/store.js +67 -9
- package/dist/wire/codec.d.ts +5 -0
- package/dist/wire/codec.js +10 -0
- package/dist/wire/frame.d.ts +2 -0
- package/dist/wire/frame.js +4 -0
- package/dist/wire/msgpack.d.ts +7 -3
- package/dist/wire/msgpack.js +11 -4
- package/dist/wire/protocol.d.ts +246 -36
- package/dist/wire/protocol.js +387 -101
- package/package.json +9 -8
- package/src/blobs.ts +13 -5
- package/src/cancel.ts +44 -2
- package/src/channels.ts +21 -6
- package/src/config.ts +138 -12
- package/src/connection.ts +377 -77
- package/src/handles.ts +32 -4
- package/src/index.ts +5 -0
- package/src/jobs.ts +63 -0
- package/src/logger.ts +158 -40
- package/src/metrics.ts +46 -1
- package/src/query.ts +33 -13
- package/src/records.ts +40 -2
- package/src/runtime/addon.ts +3 -2
- package/src/runtime/bun.ts +44 -9
- package/src/runtime/node.ts +30 -5
- package/src/runtime/pipe.ts +49 -19
- package/src/runtime/shared.ts +137 -16
- package/src/runtime.ts +31 -6
- package/src/session.ts +129 -30
- package/src/sql.ts +48 -14
- package/src/store.ts +81 -14
- package/src/wire/codec.ts +11 -0
- package/src/wire/frame.ts +5 -0
- package/src/wire/msgpack.ts +11 -4
- package/src/wire/protocol.ts +404 -103
package/dist/handles.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { goesAgain } from './connection.js';
|
|
2
2
|
import { InvalidError } from './errors.js';
|
|
3
|
+
import { ConnectionLostError } from './session.js';
|
|
3
4
|
import { Reader, Writer } from './wire/msgpack.js';
|
|
4
5
|
import { Handle } from './wire/protocol.js';
|
|
5
6
|
/** Asks through `call`, the body and the answer each in bytes of their own. */
|
|
@@ -84,7 +85,9 @@ export function handleOn(connection, method, open) {
|
|
|
84
85
|
const key = `${method}:${encoded}`;
|
|
85
86
|
let opening = connection.handles.get(key);
|
|
86
87
|
if (opening === undefined) {
|
|
87
|
-
opening = connection.session.call(method, open).then(body => Handle.decode(body).handle ?? 0)
|
|
88
|
+
opening = connection.session.call(method, open).then(body => Handle.decode(body).handle ?? 0, (err) => {
|
|
89
|
+
throw unsent(err);
|
|
90
|
+
});
|
|
88
91
|
connection.handles.set(key, opening);
|
|
89
92
|
opening.catch(() => connection.handles.delete(key));
|
|
90
93
|
}
|
|
@@ -94,10 +97,30 @@ export function handleOn(connection, method, open) {
|
|
|
94
97
|
}
|
|
95
98
|
return opening;
|
|
96
99
|
}
|
|
97
|
-
/**
|
|
100
|
+
/**
|
|
101
|
+
* A connection lost while a handle opened, as the call that needed the handle
|
|
102
|
+
* meets it: the open's REQUEST had left, the call's own had not, so the call
|
|
103
|
+
* goes again on the next connection, where the handle opens again.
|
|
104
|
+
*/
|
|
105
|
+
export function unsent(err) {
|
|
106
|
+
return err instanceof ConnectionLostError && err.sent
|
|
107
|
+
? new ConnectionLostError(err.message, false)
|
|
108
|
+
: err;
|
|
109
|
+
}
|
|
110
|
+
const named = /^[a-z0-9][a-z0-9_-]{0,63}$/;
|
|
111
|
+
/**
|
|
112
|
+
* A bucket's or a queue's name, as every engine takes it: [a-z0-9][a-z0-9_-]{0,63},
|
|
113
|
+
* with the nearest name that is when there is one: `focusroom.rooms` as `focusroom_rooms`.
|
|
114
|
+
*/
|
|
98
115
|
export function checkName(name, of) {
|
|
99
|
-
if (
|
|
100
|
-
|
|
116
|
+
if (!named.test(name)) {
|
|
117
|
+
const nearest = name
|
|
118
|
+
.toLowerCase()
|
|
119
|
+
.replace(/[^a-z0-9_-]/g, '_')
|
|
120
|
+
.replace(/^[_-]+/, '')
|
|
121
|
+
.slice(0, 64);
|
|
122
|
+
const offered = named.test(nearest) ? `, as ${JSON.stringify(nearest)}` : '';
|
|
123
|
+
throw new InvalidError(`the ${of} name ${JSON.stringify(name)}: a name is [a-z0-9][a-z0-9_-]{0,63}${offered}`);
|
|
101
124
|
}
|
|
102
125
|
}
|
|
103
126
|
/** Answers what a batch's function returned, once the batch's calls have settled. */
|
package/dist/index.d.ts
CHANGED
|
@@ -3,20 +3,20 @@ export type { Body, FileBytes, FileInfo, FileListOptions, FileOptions, FilePage,
|
|
|
3
3
|
export { withSignal } from './cancel.ts';
|
|
4
4
|
export type { Channel, EachOptions, Message as ChannelMessage, Missed, SubscribeOptions, Subscription, TxChannel, } from './channels.ts';
|
|
5
5
|
export type { Clock } from './clock.ts';
|
|
6
|
-
export { type Bounds, type Changes, type Config, type ConfigOptions, type DurationBounds, duration, type EnvLayer, type EnvOptions, type Explained, type ExplainLine, env, type FileLayer, file, int, type Layer, list, live, type Overlay, oneOf, required, Secret, type Setting, secret, type UpdateOptions, type Value, type Vars, } from './config.ts';
|
|
6
|
+
export { type Bounds, type Changes, type Config, type ConfigOptions, type DurationBounds, duration, type EnvLayer, type EnvOptions, type Explained, type ExplainGiven, type ExplainLine, type ExplainOrigin, env, type FileLayer, file, int, type Layer, list, live, type Overlay, oneOf, required, Secret, type Setting, secret, type UpdateOptions, type Value, type Vars, } from './config.ts';
|
|
7
7
|
export type { Durability, SelfMetrics } from './connection.ts';
|
|
8
8
|
export { type ConsoleFormat, type ConsoleTime, secrets } from './console.ts';
|
|
9
9
|
export * from './errors.ts';
|
|
10
10
|
export type { Page } from './handles.ts';
|
|
11
|
-
export type { AddOptions, Answer, Backoff, Concurrency, Handler, Job, JobOptions, JobPage, JobState, ListOptions, Queue, QueueOptions, Run, Schedule, ScheduleOptions, ScheduleWhen, StoreTxQueue, TxQueue, Worker, WorkOptions, } from './jobs.ts';
|
|
11
|
+
export type { Adding, AddOptions, Answer, Backoff, Concurrency, Handler, Job, JobOptions, JobPage, JobState, ListOptions, Queue, QueueOptions, Run, Schedule, ScheduleOptions, ScheduleWhen, StoreTxQueue, TxQueue, Worker, WorkOptions, } from './jobs.ts';
|
|
12
12
|
export type { Bucket, BucketOptions, Counters, CountersOptions, Entry, Page as KvPage, ValueType, WriteOptions, } from './kv.ts';
|
|
13
13
|
export type { Allowance, Quota, Rate, RateLimit, RateLimitOptions, WindowUse } from './limits.ts';
|
|
14
14
|
export type { HoldOptions, Lease, Lock, LockOptions, Permit, Semaphore, SemaphoreOptions, Tried, } from './locks.ts';
|
|
15
15
|
export { type ConsoleOut, type Logger, type LoggerOptions, logger } from './logger.ts';
|
|
16
|
-
export { type Aggregate, type AggregateQuery, type Bucket as MetricsBucket, type Buckets, type Condition as MetricsCondition, type Counter, type Damage as MetricsDamage, type Description as MetricsDescription, exponentialBuckets, type Gauge, type GaugeFuncOptions, type Histogram, type HistogramOptions, type InstrumentLabels, type InstrumentOptions, type LabelPage, type Labels, type LabelValue, type Metrics, type NameInfo, type NamePage, type Newest, type Op as MetricsOp, type PageOptions, type Percentile, type PercentileBucket, type Plan, type Query as MetricsQuery, type Series, type SeriesInput, type SeriesKind, type Timer, type TimerOptions, type Timing, type ValueBucket, type ValuePage, } from './metrics.ts';
|
|
16
|
+
export { type Aggregate, type AggregateQuery, type Bucket as MetricsBucket, type Buckets, type Condition as MetricsCondition, type Counter, type Damage as MetricsDamage, type Description as MetricsDescription, exponentialBuckets, type Gauge, type GaugeFuncOptions, type Histogram, type HistogramOptions, type InstrumentLabels, type InstrumentOptions, type LabelPage, type Labels, type LabelValue, type Metrics, type NameInfo, type NamePage, type Newest, type Op as MetricsOp, type PageOptions, type Percentile, type PercentileBucket, type Plan, type PrometheusOptions, type Query as MetricsQuery, type Series, type SeriesInput, type SeriesKind, type Timer, type TimerOptions, type Timing, type ValueBucket, type ValuePage, } from './metrics.ts';
|
|
17
17
|
export type { Once, OnceOptions } from './once.ts';
|
|
18
18
|
export type { ColumnType, Query, RowPage, Table, TableOptions, UpsertOptions } from './query.ts';
|
|
19
|
-
export type { Aggregate as EventsAggregate, AllEvents, AppendOptions, Counted, Damage, Event, EventLevel, Events, EventsOptions, Fields, Level, Lines, NewEvent, Page as EventsPage, PollBatch, Query as EventsQuery, Row as EventsRow, Stats, StoreLoggerOptions, StreamInfo, TailBatch, TailQuery, Top, } from './records.ts';
|
|
19
|
+
export type { Aggregate as EventsAggregate, AllEvents, AppendOptions, Counted, Damage, Event, EventLevel, Events, EventsOptions, Fields, Level, Levels, Lines, NewEvent, Page as EventsPage, PollBatch, Query as EventsQuery, Row as EventsRow, Stats, StoreLoggerOptions, StreamInfo, TailBatch, TailQuery, Top, } from './records.ts';
|
|
20
20
|
export type { StandardSchemaV1 } from './schema.ts';
|
|
21
21
|
export { type Found as SearchFound, type Hit as SearchHit, type IndexDocument, type IndexInfo, type IndexOptions, type IndexStatus, type Query as SearchQuery, SearchIndex, search, TableIndex, type TableIndexOptions, type Vector, type Where, type WhereValue, } from './search.ts';
|
|
22
22
|
export { type Columns, type Database, type DatabaseOptions, type Done, type Migrations, type Row, Sql, type SqlArg, type SqlTag, type SqlTx, type SqlValue, type Statement, sql, } from './sql.ts';
|
package/dist/jobs.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { Link } from './connection.ts';
|
|
1
|
+
import type { Connection, Link } from './connection.ts';
|
|
2
2
|
import { type Home, type Open, type Via } from './handles.ts';
|
|
3
3
|
import { type Rate } from './limits.ts';
|
|
4
4
|
import { type StandardSchemaV1 } from './schema.ts';
|
|
@@ -239,6 +239,17 @@ export declare class Queue<T> {
|
|
|
239
239
|
* await reminders.add({ userId: 42, text: 'Call mom' }, { delay: '1h' })
|
|
240
240
|
*/
|
|
241
241
|
add(value: T, options?: AddOptions): Promise<boolean>;
|
|
242
|
+
/**
|
|
243
|
+
* A job's add for a batch of the database the queue was opened from:
|
|
244
|
+
* `db.batch` writes it with the batch's statements, so the job commits
|
|
245
|
+
* with the rows or not at all, and shares the commit with every other
|
|
246
|
+
* write of the file, where `db.tx` holds the writer and pays a sync of
|
|
247
|
+
* its own. It adds as `add` does, unless its id is taken: its `Done` in
|
|
248
|
+
* the batch's answer changed 1 job, or 0.
|
|
249
|
+
*
|
|
250
|
+
* await db.batch([sql`update notes set body = ${body} where id = ${id}`, indexing.adding({ noteId: id })])
|
|
251
|
+
*/
|
|
252
|
+
adding(value: T, options?: AddOptions): Adding;
|
|
242
253
|
/**
|
|
243
254
|
* Makes the id's job this value at this time, whatever it was: one waiting
|
|
244
255
|
* is made new, a failed one starts again, and a running one runs once more
|
|
@@ -314,6 +325,27 @@ export declare class TxQueue<T> {
|
|
|
314
325
|
/** The job under id as the transaction sees it, its own writes among it. */
|
|
315
326
|
get(id: string): Promise<Job<T> | undefined>;
|
|
316
327
|
}
|
|
328
|
+
/** A queue's add as a database's batch carries it: the queue, and the add's call but its handle. */
|
|
329
|
+
interface AddingParts {
|
|
330
|
+
name: string;
|
|
331
|
+
source: Source<unknown>;
|
|
332
|
+
call: Omit<Written<typeof JobsCall.fields>, 'handle'>;
|
|
333
|
+
}
|
|
334
|
+
/** A job's add that a database's batch carries, from `queue.adding`: it commits with the batch's statements, or not at all. */
|
|
335
|
+
export declare class Adding {
|
|
336
|
+
#private;
|
|
337
|
+
/** Comes from `queue.adding`. */
|
|
338
|
+
constructor(parts: AddingParts);
|
|
339
|
+
}
|
|
340
|
+
/**
|
|
341
|
+
* What a batch of `home` sends of an add, on the connection it goes on: the
|
|
342
|
+
* add's call, its queue's handle opened there first. A queue kept outside
|
|
343
|
+
* `home` is refused, since its job would not commit with the rows.
|
|
344
|
+
*/
|
|
345
|
+
export declare function addingIn(adding: Adding, home: Home): (connection: Connection) => Promise<{
|
|
346
|
+
method: number;
|
|
347
|
+
body: Uint8Array;
|
|
348
|
+
}>;
|
|
317
349
|
/** A write the store's transaction keeps for its commit: its method, and its call or the id it cancels. */
|
|
318
350
|
export interface JobOp {
|
|
319
351
|
source: Source<unknown>;
|
package/dist/jobs.js
CHANGED
|
@@ -288,6 +288,7 @@ async function* watchJob(source, id) {
|
|
|
288
288
|
let last;
|
|
289
289
|
for (;;) {
|
|
290
290
|
const stream = await source.link.run('read', connection => openWatch(connection, source, id));
|
|
291
|
+
const unfollow = source.link.follow(stream);
|
|
291
292
|
try {
|
|
292
293
|
for (;;) {
|
|
293
294
|
const event = await stream.next();
|
|
@@ -309,6 +310,7 @@ async function* watchJob(source, id) {
|
|
|
309
310
|
}
|
|
310
311
|
finally {
|
|
311
312
|
stream.cancel();
|
|
313
|
+
unfollow();
|
|
312
314
|
}
|
|
313
315
|
}
|
|
314
316
|
}
|
|
@@ -356,6 +358,20 @@ export class Queue {
|
|
|
356
358
|
};
|
|
357
359
|
return (await this.#changed(methods['jobs.add'], fields)) === true;
|
|
358
360
|
}
|
|
361
|
+
/**
|
|
362
|
+
* A job's add for a batch of the database the queue was opened from:
|
|
363
|
+
* `db.batch` writes it with the batch's statements, so the job commits
|
|
364
|
+
* with the rows or not at all, and shares the commit with every other
|
|
365
|
+
* write of the file, where `db.tx` holds the writer and pays a sync of
|
|
366
|
+
* its own. It adds as `add` does, unless its id is taken: its `Done` in
|
|
367
|
+
* the batch's answer changed 1 job, or 0.
|
|
368
|
+
*
|
|
369
|
+
* await db.batch([sql`update notes set body = ${body} where id = ${id}`, indexing.adding({ noteId: id })])
|
|
370
|
+
*/
|
|
371
|
+
adding(value, options = {}) {
|
|
372
|
+
const call = { ...jobFields(options), id: options.id, value: this.#source.values.encode(value) };
|
|
373
|
+
return new Adding({ name: this.name, source: this.#source, call });
|
|
374
|
+
}
|
|
359
375
|
/**
|
|
360
376
|
* Makes the id's job this value at this time, whatever it was: one waiting
|
|
361
377
|
* is made new, a failed one starts again, and a running one runs once more
|
|
@@ -483,6 +499,35 @@ export class TxQueue {
|
|
|
483
499
|
return this.#queue.get(id);
|
|
484
500
|
}
|
|
485
501
|
}
|
|
502
|
+
// reads an add's parts, which only this module sees
|
|
503
|
+
let partsOf;
|
|
504
|
+
/** A job's add that a database's batch carries, from `queue.adding`: it commits with the batch's statements, or not at all. */
|
|
505
|
+
export class Adding {
|
|
506
|
+
#parts;
|
|
507
|
+
/** Comes from `queue.adding`. */
|
|
508
|
+
constructor(parts) {
|
|
509
|
+
this.#parts = parts;
|
|
510
|
+
}
|
|
511
|
+
static {
|
|
512
|
+
partsOf = adding => adding.#parts;
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
/**
|
|
516
|
+
* What a batch of `home` sends of an add, on the connection it goes on: the
|
|
517
|
+
* add's call, its queue's handle opened there first. A queue kept outside
|
|
518
|
+
* `home` is refused, since its job would not commit with the rows.
|
|
519
|
+
*/
|
|
520
|
+
export function addingIn(adding, home) {
|
|
521
|
+
const { name, source, call } = partsOf(adding);
|
|
522
|
+
if (source.home !== home) {
|
|
523
|
+
const kept = source.home === undefined ? 'jobs.db' : source.home.describe;
|
|
524
|
+
throw new InvalidError(`${home.describe}: jobs queue ${name}: kept in ${kept}, outside this batch of ${home.describe}`);
|
|
525
|
+
}
|
|
526
|
+
return async (connection) => {
|
|
527
|
+
const handle = await handleOn(connection, source.openMethod, source.open);
|
|
528
|
+
return { method: methods['jobs.add'], body: JobsCall.encode({ ...call, handle }) };
|
|
529
|
+
};
|
|
530
|
+
}
|
|
486
531
|
/**
|
|
487
532
|
* A queue's writes inside the store's transaction, from `tx.with(queue)` in
|
|
488
533
|
* `store.tx`: each is kept for the commit, which makes them all or none, so
|
package/dist/logger.d.ts
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { DropNotice, FailureLog, type Say } from './background.ts';
|
|
2
2
|
import { type EnvLayer, type Setting } from './config.ts';
|
|
3
3
|
import { type ConsoleFormat, type ConsoleLine, type ConsoleTime, type Redactor } from './console.ts';
|
|
4
|
-
import type { Fields, Level } from './records.ts';
|
|
4
|
+
import type { Fields, Level, Levels } from './records.ts';
|
|
5
5
|
export interface LoggerOptions {
|
|
6
6
|
/** the lines held between writes; past it a line is dropped and counted: 1024 */
|
|
7
7
|
buffer?: number;
|
|
8
|
-
/** the least level a line is
|
|
8
|
+
/** the least level a line is printed at, every level when absent */
|
|
9
9
|
level?: Level;
|
|
10
10
|
/** how a line is written as it is logged: pretty on a terminal and JSON otherwise when absent */
|
|
11
11
|
console?: ConsoleFormat;
|
|
@@ -24,12 +24,16 @@ export interface LoggerOptions {
|
|
|
24
24
|
redact?: readonly string[];
|
|
25
25
|
/** leaves a URL's password in a value as it is, which is otherwise hidden */
|
|
26
26
|
keepUrlPasswords?: boolean;
|
|
27
|
-
/**
|
|
27
|
+
/**
|
|
28
|
+
* changes each field's value before it is hidden, kept and shown, and
|
|
29
|
+
* undefined leaves the field out: return the value it was given for a
|
|
30
|
+
* field it leaves as it is
|
|
31
|
+
*/
|
|
28
32
|
replace?: (key: string, value: unknown) => unknown;
|
|
29
33
|
/**
|
|
30
|
-
* where LOG_LEVEL, LOG_FORMAT and LOG_TIME are read,
|
|
31
|
-
*
|
|
32
|
-
* reads nothing; the bare names when absent
|
|
34
|
+
* where LOG_LEVEL, LOG_CONSOLE_LEVEL, LOG_FORMAT and LOG_TIME are read,
|
|
35
|
+
* each of which wins over its option: `env('APP')` reads APP_LOG_LEVEL
|
|
36
|
+
* and the rest, and false reads nothing; the bare names when absent
|
|
33
37
|
*/
|
|
34
38
|
env?: EnvLayer | false;
|
|
35
39
|
/** adds where each line was logged: source={"function":…,"file":…,"line":…} */
|
|
@@ -46,7 +50,10 @@ interface Shared {
|
|
|
46
50
|
append: ((lines: ConsoleLine[]) => Promise<void>) | undefined;
|
|
47
51
|
stream: string;
|
|
48
52
|
most: number;
|
|
49
|
-
least
|
|
53
|
+
/** the least level kept, which the config log changes while the logger runs */
|
|
54
|
+
kept: number;
|
|
55
|
+
/** the least level printed, LOG_CONSOLE_LEVEL's or level's console part; undefined prints what is kept */
|
|
56
|
+
printed: number | undefined;
|
|
50
57
|
redact: Redactor | undefined;
|
|
51
58
|
replace: ((key: string, value: unknown) => unknown) | undefined;
|
|
52
59
|
source: boolean;
|
|
@@ -88,8 +95,12 @@ type LogDeclaration = {
|
|
|
88
95
|
format: Setting<string>;
|
|
89
96
|
time: Setting<string>;
|
|
90
97
|
};
|
|
98
|
+
/** What a logger is told: a logger's options, a store's logger's level in its two parts. */
|
|
99
|
+
type Told = Omit<LoggerOptions, 'level'> & {
|
|
100
|
+
level?: Level | Levels | undefined;
|
|
101
|
+
};
|
|
91
102
|
/** A store's logger of one stream, and what it declares its config `log` with. */
|
|
92
|
-
export declare function newStoreLogger(append: (lines: ConsoleLine[]) => Promise<void>, stream: string, options?:
|
|
103
|
+
export declare function newStoreLogger(append: (lines: ConsoleLine[]) => Promise<void>, stream: string, options?: Told): [Logger, LogConfig];
|
|
93
104
|
/** TinyStore's own lines, on the console of a logger of stream tinystore made when first needed. */
|
|
94
105
|
export declare const sayOwn: Say;
|
|
95
106
|
/**
|
|
@@ -129,8 +140,9 @@ export declare class Logger {
|
|
|
129
140
|
}
|
|
130
141
|
/**
|
|
131
142
|
* Fields as key, JSON pairs, as a console writes them and the store keeps
|
|
132
|
-
* them:
|
|
133
|
-
*
|
|
143
|
+
* them: a field whose value is undefined left out, as JSON.stringify leaves
|
|
144
|
+
* it out of an object, an Error's value its stack, which JSON would write as
|
|
145
|
+
* {}, a secret's value hidden at any depth, and a URL's password unless kept.
|
|
134
146
|
*/
|
|
135
147
|
export declare function encodeFields(fields: Fields, redact: Redactor | undefined, replace?: (key: string, value: unknown) => unknown): string[];
|
|
136
148
|
export {};
|
package/dist/logger.js
CHANGED
|
@@ -35,10 +35,26 @@ export function logger(stream, options) {
|
|
|
35
35
|
export function newLogger(append, stream, options = {}, out) {
|
|
36
36
|
return new Logger(sharedOf(append, stream, options, out).shared, []);
|
|
37
37
|
}
|
|
38
|
+
/**
|
|
39
|
+
* A level's two parts, the least kept and the least printed: one level is
|
|
40
|
+
* the kept alone, which the console then follows.
|
|
41
|
+
*/
|
|
42
|
+
function partsOf(level) {
|
|
43
|
+
if (typeof level !== 'object') {
|
|
44
|
+
return [level, undefined];
|
|
45
|
+
}
|
|
46
|
+
for (const part of Object.keys(level)) {
|
|
47
|
+
if (part !== 'store' && part !== 'console') {
|
|
48
|
+
throw new InvalidError(`a logger's level of ${JSON.stringify(part)}: its parts are store and console`);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
return [level.store, level.console];
|
|
52
|
+
}
|
|
38
53
|
/** A store's logger of one stream, and what it declares its config `log` with. */
|
|
39
54
|
export function newStoreLogger(append, stream, options = {}) {
|
|
40
55
|
const { shared, spelled } = sharedOf(append, stream, options, undefined);
|
|
41
|
-
const
|
|
56
|
+
const [kept] = partsOf(options.level);
|
|
57
|
+
const level = kept === undefined ? 'trace' : nameOf(levelOf(kept));
|
|
42
58
|
const format = options.console ?? (prettyUntold(options, undefined) ? 'pretty' : 'json');
|
|
43
59
|
const time = options.time ?? 'clock';
|
|
44
60
|
const others = (all, one) => all.filter(name => name !== one);
|
|
@@ -48,9 +64,10 @@ export function newStoreLogger(append, stream, options = {}) {
|
|
|
48
64
|
time: oneOf(time, others(times, time)),
|
|
49
65
|
};
|
|
50
66
|
const env = options.env ?? environmentOf('');
|
|
67
|
+
// the level kept, and printed too while the level has no console part
|
|
51
68
|
const take = (name) => {
|
|
52
69
|
if (Object.hasOwn(levels, name)) {
|
|
53
|
-
shared.
|
|
70
|
+
shared.kept = levels[name];
|
|
54
71
|
}
|
|
55
72
|
};
|
|
56
73
|
if (env === false) {
|
|
@@ -82,7 +99,8 @@ function sharedOf(append, stream, options, out) {
|
|
|
82
99
|
append,
|
|
83
100
|
stream,
|
|
84
101
|
most,
|
|
85
|
-
|
|
102
|
+
kept: read.kept === undefined ? Number.NEGATIVE_INFINITY : levelOf(read.kept),
|
|
103
|
+
printed: read.printed === undefined ? undefined : levelOf(read.printed),
|
|
86
104
|
redact: redactor(read.redact, read.keepUrlPasswords),
|
|
87
105
|
replace: read.replace,
|
|
88
106
|
source: read.source === true,
|
|
@@ -98,12 +116,16 @@ function sharedOf(append, stream, options, out) {
|
|
|
98
116
|
return { shared, spelled };
|
|
99
117
|
}
|
|
100
118
|
/**
|
|
101
|
-
* LOG_LEVEL, LOG_FORMAT and LOG_TIME over the options,
|
|
102
|
-
* names, as Go and Python read them;
|
|
103
|
-
*
|
|
119
|
+
* LOG_LEVEL, LOG_CONSOLE_LEVEL, LOG_FORMAT and LOG_TIME over the options,
|
|
120
|
+
* each over its own, after the prefix env names, as Go and Python read them;
|
|
121
|
+
* a console turned off stays off. Each variable read is spelled as the
|
|
122
|
+
* config `log` reads it, by its name, but LOG_CONSOLE_LEVEL, which no
|
|
123
|
+
* setting of the config holds.
|
|
104
124
|
*/
|
|
105
125
|
function fromEnvironment(options) {
|
|
106
|
-
const
|
|
126
|
+
const { level: given, ...rest } = options;
|
|
127
|
+
const [kept, printed] = partsOf(given);
|
|
128
|
+
const read = { ...rest, kept, printed };
|
|
107
129
|
const ignored = [];
|
|
108
130
|
const spelled = {};
|
|
109
131
|
const env = options.env ?? environmentOf('');
|
|
@@ -112,16 +134,14 @@ function fromEnvironment(options) {
|
|
|
112
134
|
}
|
|
113
135
|
const variables = environment(env);
|
|
114
136
|
const name = (variable) => (env.prefix === '' ? variable : `${env.prefix}_${variable}`);
|
|
115
|
-
const level = variables
|
|
137
|
+
const level = levelIn(variables, name('LOG_LEVEL'), ignored);
|
|
116
138
|
if (level !== undefined) {
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
ignored.push([name('LOG_LEVEL'), level, 'trace, debug, info, warn or error']);
|
|
124
|
-
}
|
|
139
|
+
read.kept = levels[level];
|
|
140
|
+
spelled[name('LOG_LEVEL')] = level;
|
|
141
|
+
}
|
|
142
|
+
const shown = levelIn(variables, name('LOG_CONSOLE_LEVEL'), ignored);
|
|
143
|
+
if (shown !== undefined) {
|
|
144
|
+
read.printed = levels[shown];
|
|
125
145
|
}
|
|
126
146
|
const format = variables(name('LOG_FORMAT'));
|
|
127
147
|
if (format !== undefined) {
|
|
@@ -147,6 +167,19 @@ function fromEnvironment(options) {
|
|
|
147
167
|
}
|
|
148
168
|
return [read, ignored, spelled];
|
|
149
169
|
}
|
|
170
|
+
/** The level a variable names, the case ignored and warning read as warn; a value it cannot mean is ignored. */
|
|
171
|
+
function levelIn(variables, name, ignored) {
|
|
172
|
+
const text = variables(name);
|
|
173
|
+
if (text === undefined) {
|
|
174
|
+
return undefined;
|
|
175
|
+
}
|
|
176
|
+
const known = text.toLowerCase() === 'warning' ? 'warn' : text.toLowerCase();
|
|
177
|
+
if (Object.hasOwn(levels, known)) {
|
|
178
|
+
return known;
|
|
179
|
+
}
|
|
180
|
+
ignored.push([name, text, 'trace, debug, info, warn or error']);
|
|
181
|
+
return undefined;
|
|
182
|
+
}
|
|
150
183
|
/** The variables of env's files, the process's or `from`'s over them, a missing file skipped; an empty one is none. */
|
|
151
184
|
function environment(env) {
|
|
152
185
|
const fromFiles = {};
|
|
@@ -301,11 +334,13 @@ export class Logger {
|
|
|
301
334
|
}
|
|
302
335
|
}
|
|
303
336
|
#log(level, message, attrs) {
|
|
304
|
-
|
|
337
|
+
// a line that no place of the logger takes is never encoded
|
|
338
|
+
if (!prints(this.#shared, level) && !keeps(this.#shared, level)) {
|
|
305
339
|
return;
|
|
306
340
|
}
|
|
307
341
|
this.#emit({ level, msg: message }, attrs);
|
|
308
342
|
}
|
|
343
|
+
/** Prints a line from the level printed and keeps it from the level kept; an event, which has none, both. */
|
|
309
344
|
#emit(what, attrs) {
|
|
310
345
|
const trace = traceOf(currentTrace());
|
|
311
346
|
const shared = this.#shared;
|
|
@@ -318,8 +353,12 @@ export class Logger {
|
|
|
318
353
|
attrs: shared.source ? [...encoded, sourceKey, sourceOfCaller()] : encoded,
|
|
319
354
|
...trace,
|
|
320
355
|
};
|
|
321
|
-
|
|
322
|
-
|
|
356
|
+
if (what.level === undefined || prints(shared, what.level)) {
|
|
357
|
+
shared.print?.(line);
|
|
358
|
+
}
|
|
359
|
+
if (what.level === undefined || keeps(shared, what.level)) {
|
|
360
|
+
this.#queue(line);
|
|
361
|
+
}
|
|
323
362
|
}
|
|
324
363
|
#queue(line) {
|
|
325
364
|
const shared = this.#shared;
|
|
@@ -341,6 +380,14 @@ export class Logger {
|
|
|
341
380
|
}
|
|
342
381
|
}
|
|
343
382
|
}
|
|
383
|
+
/** Whether a logger prints a line of level: it has a console, and the level is the level printed or above. */
|
|
384
|
+
function prints(shared, level) {
|
|
385
|
+
return shared.print !== undefined && level >= (shared.printed ?? shared.kept);
|
|
386
|
+
}
|
|
387
|
+
/** Whether a logger keeps a line of level: it is a store's, and the level is its level or above. */
|
|
388
|
+
function keeps(shared, level) {
|
|
389
|
+
return shared.append !== undefined && level >= shared.kept;
|
|
390
|
+
}
|
|
344
391
|
/**
|
|
345
392
|
* Writes what the buffer holds, and again while half of it filled during the
|
|
346
393
|
* write before. A full buffer drops a line before the line could ask for a
|
|
@@ -350,21 +397,60 @@ export class Logger {
|
|
|
350
397
|
async function writeWaiting(shared, append) {
|
|
351
398
|
let batch = shared.waiting.splice(0);
|
|
352
399
|
while (batch.length > 0) {
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
400
|
+
for (const piece of pieces(batch)) {
|
|
401
|
+
try {
|
|
402
|
+
await append(piece);
|
|
403
|
+
shared.failures?.succeeded();
|
|
404
|
+
}
|
|
405
|
+
catch (err) {
|
|
406
|
+
shared.dropped += piece.length;
|
|
407
|
+
shared.failures?.failed(err);
|
|
408
|
+
}
|
|
360
409
|
}
|
|
361
410
|
batch = shared.waiting.length >= shared.most / 2 ? shared.waiting.splice(0) : [];
|
|
362
411
|
}
|
|
363
412
|
}
|
|
413
|
+
/**
|
|
414
|
+
* The text a write carries at most. A request's body holds 2 MiB, and a
|
|
415
|
+
* buffer of long lines, or one made larger, held more: the whole write was
|
|
416
|
+
* refused and every line in it dropped. Counted in UTF-16 units, three bytes
|
|
417
|
+
* each at worst, so a piece stays within the body however its text is spelt.
|
|
418
|
+
*/
|
|
419
|
+
const PIECE = 512 * 1024;
|
|
420
|
+
/** The lines in writes of at most PIECE of text each, in their order. */
|
|
421
|
+
function pieces(lines) {
|
|
422
|
+
const out = [];
|
|
423
|
+
let piece = [];
|
|
424
|
+
let size = 0;
|
|
425
|
+
for (const line of lines) {
|
|
426
|
+
const weight = textOf(line);
|
|
427
|
+
if (piece.length > 0 && size + weight > PIECE) {
|
|
428
|
+
out.push(piece);
|
|
429
|
+
piece = [];
|
|
430
|
+
size = 0;
|
|
431
|
+
}
|
|
432
|
+
piece.push(line);
|
|
433
|
+
size += weight;
|
|
434
|
+
}
|
|
435
|
+
if (piece.length > 0) {
|
|
436
|
+
out.push(piece);
|
|
437
|
+
}
|
|
438
|
+
return out;
|
|
439
|
+
}
|
|
440
|
+
/** A line's text, with a little for its time, level and framing. */
|
|
441
|
+
function textOf(line) {
|
|
442
|
+
let size = 64 + (line.msg?.length ?? 0) + (line.event?.length ?? 0);
|
|
443
|
+
for (const part of line.attrs)
|
|
444
|
+
size += part.length;
|
|
445
|
+
for (const part of line.context)
|
|
446
|
+
size += part.length;
|
|
447
|
+
return size;
|
|
448
|
+
}
|
|
364
449
|
/**
|
|
365
450
|
* Fields as key, JSON pairs, as a console writes them and the store keeps
|
|
366
|
-
* them:
|
|
367
|
-
*
|
|
451
|
+
* them: a field whose value is undefined left out, as JSON.stringify leaves
|
|
452
|
+
* it out of an object, an Error's value its stack, which JSON would write as
|
|
453
|
+
* {}, a secret's value hidden at any depth, and a URL's password unless kept.
|
|
368
454
|
*/
|
|
369
455
|
export function encodeFields(fields, redact, replace) {
|
|
370
456
|
const pairs = Array.isArray(fields)
|
|
@@ -373,6 +459,10 @@ export function encodeFields(fields, redact, replace) {
|
|
|
373
459
|
const flat = [];
|
|
374
460
|
for (const [key, given] of pairs) {
|
|
375
461
|
const value = replace === undefined ? given : replace(key, given);
|
|
462
|
+
// before it is hidden, as Rust's replace drops a field: a secret given no value is no field
|
|
463
|
+
if (value === undefined) {
|
|
464
|
+
continue;
|
|
465
|
+
}
|
|
376
466
|
const shown = value instanceof Error ? (value.stack ?? String(value)) : value;
|
|
377
467
|
if (redact?.hides(key)) {
|
|
378
468
|
flat.push(key, redacted);
|
package/dist/metrics.d.ts
CHANGED
|
@@ -42,6 +42,15 @@ export interface SeriesInput {
|
|
|
42
42
|
/** how long the name's samples stay: its own, kept in the file; unless given, what it keeps already */
|
|
43
43
|
keep?: Duration | undefined;
|
|
44
44
|
}
|
|
45
|
+
/** What an ingest of an exporter's text gives every series of it. */
|
|
46
|
+
export interface PrometheusOptions {
|
|
47
|
+
/** labels every series gets beside its own, such as the instance scraped; one the text gives too is refused, naming its line */
|
|
48
|
+
labels?: Record<string, LabelValue> | undefined;
|
|
49
|
+
/** the time of every sample the text gives none; the store's clock unless given */
|
|
50
|
+
at?: Time | undefined;
|
|
51
|
+
/** how long every name of the text keeps its samples */
|
|
52
|
+
keep?: Duration | undefined;
|
|
53
|
+
}
|
|
45
54
|
/** A series as a read gives it: its samples as two columns. */
|
|
46
55
|
export interface Series {
|
|
47
56
|
name: string;
|
|
@@ -118,6 +127,7 @@ export interface Plan {
|
|
|
118
127
|
bytes: number;
|
|
119
128
|
decoded: number;
|
|
120
129
|
exceeds: {
|
|
130
|
+
/** series, bytes read, samples decoded or answer */
|
|
121
131
|
limit: string;
|
|
122
132
|
wanted: number;
|
|
123
133
|
bound: number;
|
|
@@ -177,6 +187,15 @@ export declare class Metrics {
|
|
|
177
187
|
constructor(link: Link, instruments: Instruments);
|
|
178
188
|
/** Writes series and their samples, every one or none: a refused series names itself. */
|
|
179
189
|
ingest(series: SeriesInput | readonly SeriesInput[]): Promise<void>;
|
|
190
|
+
/**
|
|
191
|
+
* Reads the text an exporter serves, in Prometheus' format or OpenMetrics', and writes its
|
|
192
|
+
* series, every one or none: a counter as a counter, a histogram's _bucket series as
|
|
193
|
+
* histogram. A line it cannot read is refused, naming the line.
|
|
194
|
+
*
|
|
195
|
+
* const text = await (await fetch('http://web-1:9100/metrics')).text()
|
|
196
|
+
* await store.metrics.ingestPrometheus(text, { labels: { instance: 'web-1' } })
|
|
197
|
+
*/
|
|
198
|
+
ingestPrometheus(text: string | Uint8Array, options?: PrometheusOptions): Promise<void>;
|
|
180
199
|
/** The samples of each series the query finds, in its range: whole, or a limit error naming its bound. */
|
|
181
200
|
read(query: Query): Promise<Series[]>;
|
|
182
201
|
/** The newest sample of each series the query finds, when it has one in the range. */
|
package/dist/metrics.js
CHANGED
|
@@ -8,7 +8,7 @@ import { download } from './connection.js';
|
|
|
8
8
|
import { InvalidError } from './errors.js';
|
|
9
9
|
import { Counter, Gauge, Histogram, labelsOf, Timer, } from './instruments.js';
|
|
10
10
|
import { ms, unixMs } from './time.js';
|
|
11
|
-
import { Empty, MetricsAggregateAsked, MetricsAggregates, MetricsDamages, MetricsDeleted, MetricsDescription, MetricsIngest, MetricsLatest, MetricsNameAsked, MetricsNamePage, MetricsPageAsked, MetricsPlan, MetricsQuery, MetricsRead, MetricsSeriesNamed, MetricsTextPage, methods, } from './wire/protocol.js';
|
|
11
|
+
import { Empty, MetricsAggregateAsked, MetricsAggregates, MetricsDamages, MetricsDeleted, MetricsDescription, MetricsExposition, MetricsIngest, MetricsLatest, MetricsNameAsked, MetricsNamePage, MetricsPageAsked, MetricsPlan, MetricsQuery, MetricsRead, MetricsSeriesNamed, MetricsTextPage, methods, } from './wire/protocol.js';
|
|
12
12
|
export { Counter, exponentialBuckets, Gauge, Histogram, Timer, Timing, } from './instruments.js';
|
|
13
13
|
/**
|
|
14
14
|
* A store's metrics: series by name and labels, their reads and aggregates,
|
|
@@ -30,6 +30,27 @@ export class Metrics {
|
|
|
30
30
|
const body = MetricsIngest.encode({ series: given.map(newSeries) });
|
|
31
31
|
await this.#link.run('write', connection => connection.session.call(methods['metrics.ingest'], body));
|
|
32
32
|
}
|
|
33
|
+
/**
|
|
34
|
+
* Reads the text an exporter serves, in Prometheus' format or OpenMetrics', and writes its
|
|
35
|
+
* series, every one or none: a counter as a counter, a histogram's _bucket series as
|
|
36
|
+
* histogram. A line it cannot read is refused, naming the line.
|
|
37
|
+
*
|
|
38
|
+
* const text = await (await fetch('http://web-1:9100/metrics')).text()
|
|
39
|
+
* await store.metrics.ingestPrometheus(text, { labels: { instance: 'web-1' } })
|
|
40
|
+
*/
|
|
41
|
+
async ingestPrometheus(text, options = {}) {
|
|
42
|
+
const labels = {};
|
|
43
|
+
for (const [label, value] of Object.entries(options.labels ?? {})) {
|
|
44
|
+
labels[label] = String(value);
|
|
45
|
+
}
|
|
46
|
+
const body = MetricsExposition.encode({
|
|
47
|
+
text: typeof text === 'string' ? new TextEncoder().encode(text) : text,
|
|
48
|
+
labels,
|
|
49
|
+
at: options.at === undefined ? undefined : unixMs(options.at),
|
|
50
|
+
keep: options.keep === undefined ? undefined : ms(options.keep),
|
|
51
|
+
});
|
|
52
|
+
await this.#link.run('write', connection => connection.session.call(methods['metrics.ingestPrometheus'], body));
|
|
53
|
+
}
|
|
33
54
|
/** The samples of each series the query finds, in its range: whole, or a limit error naming its bound. */
|
|
34
55
|
async read(query) {
|
|
35
56
|
const parts = await this.#download('metrics.read', MetricsQuery.encode(queryOf(query)));
|