@toa.io/extensions.chronicles 0.0.0-stage → 1.0.0-alpha.333

package/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2020-present Artem Gurtovoi
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/components/chronicles.encounters/manifest.toa.yaml

@@ -0,0 +1,108 @@
+namespace: chronicles
+name: encounters
+
+# The extension writes what this component is sent to `encounters`.
+chronicles: ~
+
+# What clients send, by name: documentation for whoever reads the log or writes a client, which
+# nothing sent is checked against.
+configuration:
+  schema:
+    type: object
+    propertyNames:
+      pattern: ^[a-z][a-z0-9_]{0,63}$
+    additionalProperties:
+      type: object
+      properties:
+        description:
+          description: What the client tells by sending it
+          type: string
+        fields:
+          description: What each field of its payload is, by path
+          type: object
+          additionalProperties:
+            type: string
+      required: [description]
+      additionalProperties: false
+  defaults: {}
+
+operations:
+  anonymous:
+    type: effect
+    input: &batch
+      description: >
+        `{ sent, visitor?, session?, events }` — `sent` when the client sends the batch, by its
+        clock — at most 100 events, each `{ id, name, time, payload? }`, `time` when it
+        happened, by the same clock; and the client's `ip`, `country`, `origin` and `agent`,
+        which the gateway puts in; as JSON, or as the form `navigator.sendBeacon` sends, whose
+        `events` is their JSON and `sent` its digits, which the extension parses and checks
+      type: object
+    output: &accepted
+      type: object
+      properties:
+        status:
+          type: integer
+        accepted:
+          type: integer
+    errors: &errors [INVALID, OVERSIZED]
+  identified:
+    type: effect
+    input: *batch
+    output: *accepted
+    errors: *errors
+  describe:
+    type: computation
+    output:
+      description: What clients send, by name, as the configuration documents it
+      type: object
+
+exposition:
+  isolated: true
+  /:
+    POST:
+      anonymous: true
+      map:ip: ip
+      map:country: country
+      map:headers:
+        origin: origin
+        agent: user-agent
+      io:throttle:
+        key: [ip]
+        requests: 60
+        interval: 60
+      io:status: status
+      io:output: [status, accepted]
+      help:method: null
+      endpoint: anonymous
+    GET:
+      auth:role: system:chronicles:encounters
+      mcp:tool: true
+      help:method:
+        title: What clients send
+        description: >
+          The encounters clients send, by name: what each tells, and what each field of its
+          payload is. It documents, and holds nothing back: a name it does not tell is written
+          as any other. A client sends a batch with `sent`, its clock as it sends, and each event
+          with `time`, its clock as the event happened — both read as they are, never corrected,
+          however wrong the clock is: what is kept is when the gateway received the batch, less
+          the distance between the two.
+      io:output: true
+      endpoint: describe
+  /:identity:
+    POST:
+      auth:id: identity
+      map:segments:
+        identity: identity
+      map:ip: ip
+      map:country: country
+      map:headers:
+        origin: origin
+        agent: user-agent
+      io:throttle:
+        key: [identity]
+        requests: 60
+        interval: 60
+      io:status: status
+      io:output: [status, accepted]
+      help:method: null
+      endpoint: identified

package/components/chronicles.encounters/operations/anonymous.js

@@ -0,0 +1,4 @@
+// what anyone sent, checked and written by the extension
+export async function effect(input, context) {
+  return await context.chronicles.write(input, null)
+}

package/components/chronicles.encounters/operations/describe.js

@@ -0,0 +1,4 @@
+// what the context documents of what clients send
+export function computation(_, context) {
+  return context.configuration
+}

package/components/chronicles.encounters/operations/identified.js

@@ -0,0 +1,4 @@
+// what a caller the gateway verified sent, written as that identity's
+export async function effect(input, context) {
+  return await context.chronicles.write(input, input.identity)
+}

package/components/chronicles.encounters/types/index.d.ts

@@ -0,0 +1,3 @@
+export * from './toa.d.ts'
+
+// What a manifest does not state belongs here, and every run keeps it.

package/components/chronicles.encounters/types/toa.d.ts

@@ -0,0 +1,36 @@
+// Written by `toa types`. Every run rewrites it.
+// What a manifest does not state belongs in a file of your own.
+
+import type { CodedError } from '@toa.io/core/types'
+import type { Chronicles } from '@toa.io/extensions.chronicles'
+import type { FetchInit } from '@toa.io/extensions.fetch'
+import type { Logs, Metrics, Span } from '@toa.io/extensions.telemetry'
+
+export type AnonymousInput = Record<string, unknown>
+
+export type AnonymousOutput = {
+  status?: number
+  accepted?: number
+}
+
+export type IdentifiedInput = Record<string, unknown>
+
+export type IdentifiedOutput = {
+  status?: number
+  accepted?: number
+}
+
+export type DescribeOutput = Record<string, unknown>
+
+export interface Component {
+  anonymous: (request: { input: AnonymousInput, task?: boolean }) => Promise<AnonymousOutput | CodedError<"INVALID" | "OVERSIZED">>
+  identified: (request: { input: IdentifiedInput, task?: boolean }) => Promise<IdentifiedOutput | CodedError<"INVALID" | "OVERSIZED">>
+  describe: (request: { input?: null, task?: boolean }) => Promise<DescribeOutput>
+}
+
+export type Configuration = Record<string, {
+  /** What the client tells by sending it */
+  description: string
+  /** What each field of its payload is, by path */
+  fields?: Record<string, string>
+}>

package/components/chronicles.queries/manifest.toa.yaml

@@ -0,0 +1,139 @@
+namespace: chronicles
+name: queries
+
+# The extension puts the log's reader on this component.
+chronicles: ~
+
+operations:
+  describe:
+    type: computation
+    output:
+      type: object
+      properties:
+        guide:
+          description: How to read the log, and how its store is read
+          type: string
+        labels:
+          type: array
+        states:
+          type: array
+        encounters:
+          type: array
+        views:
+          type: array
+  query:
+    type: computation
+    input:
+      type: object
+      properties:
+        sql:
+          description: One statement over the log, as the guide `describe` answers says
+          type: string
+          minLength: 1
+          maxLength: 65536
+      required: [sql]
+    output:
+      type: object
+      properties:
+        columns:
+          type: array
+        rows:
+          type: array
+        truncated:
+          type: boolean
+    errors: [REFUSED]
+  define:
+    type: effect
+    input:
+      type: object
+      properties:
+        name: &name
+          description: The view's name, which is what a query reads it by
+          type: string
+          pattern: ^[a-z][a-z0-9_]{0,62}$
+        sql:
+          description: The statement the view is, one a query could run
+          type: string
+          minLength: 1
+          maxLength: 65536
+        refresh:
+          description: Seconds between computations of a materialized view; absent, a plain view
+          type: integer
+          minimum: 3600
+        title:
+          description: What the view is, in a few words
+          type: string
+          minLength: 1
+          maxLength: 100
+        description:
+          description: What a row of the view is, and what each of its columns counts
+          type: string
+          minLength: 1
+          maxLength: 4096
+      required: [name, sql]
+    errors: [REFUSED, RESERVED, EXHAUSTED]
+  drop:
+    type: effect
+    input:
+      type: object
+      properties:
+        name: *name
+      required: [name]
+    errors: [NOT_FOUND]
+
+exposition:
+  isolated: true
+  /:
+    auth:role: system:chronicles:query
+    mcp:tool: true
+    GET:
+      help:method:
+        title: What the log holds
+        description: >
+          A guide to reading the log and its store, to read before a query — what each table and
+          column means, and which time is which; the labels of `events` the components declare,
+          each with what it means, the view that reads it in types, how many rows it has, the
+          first and last time of them, and its fields with their types and what they mean; the
+          components with a state in `states`, likewise; the names of `encounters` clients sent,
+          with when the newest batch was received, the fields of their payloads as they were sent
+          and what the context documents of them; and the views defined over the log, with their
+          titles, descriptions and when a materialized one was last refreshed. `states` is each
+          entity as it stands now: what it was before is read from its events.
+      io:output: true
+      endpoint: describe
+    POST:
+      help:method:
+        title: Query the log
+        description: >
+          One statement, as the guide `chronicles.queries.GET` answers says, over `events`, a row
+          for everything a component did (id, label, component, time, entity, version, identity,
+          trace, payload), `states`, each entity as it stands (component, entity, version,
+          identity, time, payload), `encounters`, what clients sent (id, name, time, received,
+          visitor, session, identity, ip, country, origin, agent, browser, os, device, payload),
+          the view of each declared label and state, and the views. Every `time` is when it
+          happened, by the server: a change when the component made it, a fact when it was sent,
+          an encounter when the gateway received its batch less how long before sending the
+          client measured it happened — never the client's clock as it reads, nor when the store
+          wrote the row. An encounter's `received` is when the gateway received its batch. At
+          most 30 seconds, 10000 rows and 64 MiB; at the rows, `truncated` is true.
+      io:output: true
+      endpoint: query
+  /views/:name:
+    auth:role: system:chronicles:views
+    mcp:tool: true
+    PUT:
+      help:method:
+        title: Define a view
+        description: >
+          A view over the log, by a statement a query could run, with a title and a description
+          that tell whoever finds it what it counts. Without `refresh` it is computed when it is
+          read; with `refresh`, in seconds and at least 3600, it is computed on that cycle and
+          read as a table. Defining a name again replaces the view.
+      help:route:
+        name: The view's name
+      endpoint: define
+    DELETE:
+      help:method: Drop a view
+      help:route:
+        name: The view's name
+      endpoint: drop

package/components/chronicles.queries/operations/define.js

@@ -0,0 +1,3 @@
+export async function effect(input, context) {
+  return await context.chronicles.define(input)
+}

package/components/chronicles.queries/operations/describe.js

@@ -0,0 +1,3 @@
+export async function computation(_, context) {
+  return await context.chronicles.describe()
+}

package/components/chronicles.queries/operations/drop.js

@@ -0,0 +1,3 @@
+export async function effect(input, context) {
+  return await context.chronicles.drop(input.name)
+}

package/components/chronicles.queries/operations/query.js

@@ -0,0 +1,3 @@
+export async function computation(input, context) {
+  return await context.chronicles.query(input.sql)
+}

package/components/chronicles.queries/types/index.d.ts

@@ -0,0 +1,3 @@
+export * from './toa.d.ts'
+
+// What a manifest does not state belongs here, and every run keeps it.

package/components/chronicles.queries/types/toa.d.ts

@@ -0,0 +1,49 @@
+// Written by `toa types`. Every run rewrites it.
+// What a manifest does not state belongs in a file of your own.
+
+import type { CodedError } from '@toa.io/core/types'
+
+export type DescribeOutput = {
+  /** How to read the log, and how its store is read */
+  guide?: string
+  labels?: unknown[]
+  states?: unknown[]
+  encounters?: unknown[]
+  views?: unknown[]
+}
+
+export type QueryInput = {
+  /** One statement over the log, as the guide `describe` answers says */
+  sql: string
+}
+
+export type QueryOutput = {
+  columns?: unknown[]
+  rows?: unknown[]
+  truncated?: boolean
+}
+
+export type DefineInput = {
+  /** The view's name, which is what a query reads it by */
+  name: string
+  /** The statement the view is, one a query could run */
+  sql: string
+  /** Seconds between computations of a materialized view; absent, a plain view */
+  refresh?: number
+  /** What the view is, in a few words */
+  title?: string
+  /** What a row of the view is, and what each of its columns counts */
+  description?: string
+}
+
+export type DropInput = {
+  /** The view's name, which is what a query reads it by */
+  name: string
+}
+
+export interface Component {
+  describe: (request: { input?: null, task?: boolean }) => Promise<DescribeOutput>
+  query: (request: { input: QueryInput, task?: boolean }) => Promise<QueryOutput | CodedError<"REFUSED">>
+  define: (request: { input: DefineInput, task?: boolean }) => Promise<{} | null | undefined | CodedError<"REFUSED" | "RESERVED" | "EXHAUSTED">>
+  drop: (request: { input: DropInput, task?: boolean }) => Promise<{} | null | undefined | CodedError<"NOT_FOUND">>
+}

package/components/chronicles.relay/manifest.toa.yaml

@@ -0,0 +1,43 @@
+namespace: chronicles
+name: relay
+
+# The extension publishes what this component is tasked with to the broker of the log.
+chronicles: ~
+
+operations:
+  record:
+    type: effect
+    input:
+      description: A fact a component sent, as the row it is published as
+      type: object
+      properties:
+        id:
+          type: string
+        label:
+          type: string
+        component:
+          type: string
+        time:
+          type: integer
+        entity:
+          type: "null"
+        version:
+          type: "null"
+        identity:
+          type: [string, "null"]
+        trace:
+          type: [string, "null"]
+        payload:
+          type: object
+      required: [id, label, component, time, payload]
+  encounter:
+    type: effect
+    input:
+      description: The encounters of a batch a client sent, as the rows they are published as
+      type: object
+      properties:
+        encounters:
+          type: array
+          items:
+            type: object
+      required: [encounters]

package/components/chronicles.relay/operations/encounter.js

@@ -0,0 +1,4 @@
+// the encounters of a batch a client sent, published by the extension
+export async function effect(input, context) {
+  await context.chronicles.encounter(input.encounters)
+}

package/components/chronicles.relay/operations/record.js

@@ -0,0 +1,4 @@
+// a fact a component sent with `context.chronicles.send`, published by the extension
+export async function effect(input, context) {
+  await context.chronicles.record(input)
+}

package/components/chronicles.relay/types/index.d.ts

@@ -0,0 +1,3 @@
+export * from './toa.d.ts'
+
+// What a manifest does not state belongs here, and every run keeps it.

package/components/chronicles.relay/types/toa.d.ts

@@ -0,0 +1,24 @@
+// Written by `toa types`. Every run rewrites it.
+// What a manifest does not state belongs in a file of your own.
+
+
+export type RecordInput = {
+  id: string
+  label: string
+  component: string
+  time: number
+  entity?: null
+  version?: null
+  identity?: string | null
+  trace?: string | null
+  payload: Record<string, unknown>
+}
+
+export type EncounterInput = {
+  encounters: Array<Record<string, unknown>>
+}
+
+export interface Component {
+  record: (request: { input: RecordInput, task?: boolean }) => Promise<unknown>
+  encounter: (request: { input: EncounterInput, task?: boolean }) => Promise<unknown>
+}

package/guides/clickhouse.md

@@ -0,0 +1,19 @@
+# ClickHouse
+
+Queries are ClickHouse SQL, one statement, without `FORMAT`.
+
+- Read `events`, `states` and `encounters` with `FINAL`: a row delivered twice is two until
+  ClickHouse merges them.
+- `payload` is a `JSON` column. A path is `payload.total`, `payload.shipping.country`; its type is
+  whatever was written, so cast it: `payload.total::Int64`, `toString(payload.path)`.
+  `JSONAllPaths(payload)` lists the paths of a row, `JSONAllPathsWithTypes(payload)` with types.
+- A declared field's `read` gives it in its type: an object as a named `Tuple`, read as
+  `shipping.country`, an array of objects as an `Array(Tuple(...))`, read with
+  `ARRAY JOIN items AS item` and `item.sku`. Every one is `Nullable`, and `NULL` where a row's
+  value was not of its type: `sum`, `avg` and `count(field)` skip `NULL`, `arraySum` refuses a
+  `Nullable` array — use `ARRAY JOIN` instead.
+- `time`, and an encounter's `received`, are `DateTime64(3, 'UTC')`: `toDate(time)`, `toStartOfHour(time)`,
+  `time >= now() - INTERVAL 7 DAY`.
+- A query runs at most 30 seconds and answers at most 10000 rows and 64 MiB; past the rows it is
+  `truncated`. Aggregate rather than read rows.
+- A view with `refresh` is computed on that cycle: `refreshed` says when it last was.

package/guides/log.md

@@ -0,0 +1,52 @@
+# Reading the log
+
+The log is three tables: `events`, `states` and `encounters`. Every row has `scope`, the
+application it is of; you read the rows of yours.
+
+- `events`: a row for everything a component did. `label` is `<component>.<event>` for a change,
+  with the `entity` that changed and its `version`, or `<component>.<name>` for a fact, with
+  neither. `identity` is whose the row is, where the component says; `trace` is the call that
+  caused it.
+- `states`: each entity as it stands now, by `component` and `entity`, at its latest `version`.
+  Nothing before it is kept: what an entity was at a time in the past is read from its events.
+- `encounters`: what clients sent, by `name`. `visitor` is the client's id of itself, kept across
+  sessions; `session` is one period of its activity; `identity` is set where a signed-in client
+  sent it. `ip` is the client's network, `country` its country, `origin` the site it was sent from,
+  `agent` its `user-agent`, and `browser`, `os` and `device` what the agent says it is.
+
+## Time
+
+Every row has `time`, when it happened, on the server's clock. No column is when the store wrote a
+row.
+
+- A change, in `events` and `states`: when the component made it — the entity's `UPDATED`, or the
+  property the component names, or else the moment of the change.
+- A fact: when the operation sent it.
+- An encounter: when the gateway received its batch, less how long before sending the batch the
+  client says the event happened. A client's clock may be off by minutes or years; what it
+  measures between two of its own moments is not. So `time` is the gateway's clock at the
+  distance the client measured: the order and the intervals of one client's encounters are exact,
+  and so is when they happened, to the time a request takes to arrive. A batch a client held
+  while offline is written at when its events happened.
+
+An encounter also has `received`, when the gateway received its batch: one `received` is shared by
+every encounter of a batch, and a `received` far behind now means clients stopped sending.
+
+Rows can arrive late — after the service that writes them was away — and are written at their own
+time, so a count of the last hours may still grow. Compare `time` of encounters and of events
+directly: both are the server's.
+
+A declared label's payload holds only what is declared. `chronicles.queries.GET` tells each
+declared field's type, what it means, and `read`: the expression that selects it from the table
+in its type — use it rather than casting the payload yourself. An encounter's
+payload is what the client sent, and its fields differ between versions of a client: two types of
+one field are two versions.
+
+A visitor is not a person: one identity may have many visitors, and a browser that clears its
+storage is a new one. The visitors of an identity are the rows that have both. Leave out
+`device = 'bot'` from any question of who visits. Safari on an iPad reports a desktop Mac.
+
+Nothing arrives in order: order by `time`, or by `version` within one entity. A view another agent
+defined says what it counts in its title and description; read them before reusing one, and give
+your own views both. A view is read by its name, which `chronicles.queries.GET` tells with its
+database.

package/package.json

@@ -1,6 +1,41 @@
 {
   "name": "@toa.io/extensions.chronicles",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "1.0.0-alpha.333",
+  "type": "module",
+  "description": "Toa Chronicles: a log of a context, kept in ClickHouse",
+  "author": "temich <tema.gurtovoy@gmail.com>",
+  "homepage": "https://github.com/toa-io/toa#readme",
+  "main": "transpiled/index.js",
+  "types": "transpiled/index.d.ts",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/toa-io/toa.git"
+  },
+  "bugs": {
+    "url": "https://github.com/toa-io/toa/issues"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "files": [
+    "transpiled",
+    "components",
+    "guides",
+    "store",
+    "readme.md",
+    "!**/tsconfig.tsbuildinfo",
+    "!**/*.test.*",
+    "!**/.env*"
+  ],
+  "scripts": {
+    "transpile": "tsc"
+  },
+  "dependencies": {
+    "@toa.io/core": "1.0.0-alpha.333",
+    "@toa.io/definitions": "1.0.0-alpha.333",
+    "@toa.io/generic": "1.0.0-alpha.332",
+    "comq": "0.28.0",
+    "openspan": "1.0.0-alpha.330"
+  },
+  "gitHead": "ddb74b2f99026fae540a8a8e0a49c26cbd22886a"
+}