@toa.io/extensions.chronicles 1.0.0-alpha.332 → 1.0.0-alpha.333

package/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2020-present Artem Gurtovoi
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/components/chronicles.encounters/manifest.toa.yaml

@@ -0,0 +1,108 @@
+namespace: chronicles
+name: encounters
+
+# The extension writes what this component is sent to `encounters`.
+chronicles: ~
+
+# What clients send, by name: documentation for whoever reads the log or writes a client, which
+# nothing sent is checked against.
+configuration:
+  schema:
+    type: object
+    propertyNames:
+      pattern: ^[a-z][a-z0-9_]{0,63}$
+    additionalProperties:
+      type: object
+      properties:
+        description:
+          description: What the client tells by sending it
+          type: string
+        fields:
+          description: What each field of its payload is, by path
+          type: object
+          additionalProperties:
+            type: string
+      required: [description]
+      additionalProperties: false
+  defaults: {}
+
+operations:
+  anonymous:
+    type: effect
+    input: &batch
+      description: >
+        `{ sent, visitor?, session?, events }` — `sent` when the client sends the batch, by its
+        clock — at most 100 events, each `{ id, name, time, payload? }`, `time` when it
+        happened, by the same clock; and the client's `ip`, `country`, `origin` and `agent`,
+        which the gateway puts in; as JSON, or as the form `navigator.sendBeacon` sends, whose
+        `events` is their JSON and `sent` its digits, which the extension parses and checks
+      type: object
+    output: &accepted
+      type: object
+      properties:
+        status:
+          type: integer
+        accepted:
+          type: integer
+    errors: &errors [INVALID, OVERSIZED]
+  identified:
+    type: effect
+    input: *batch
+    output: *accepted
+    errors: *errors
+  describe:
+    type: computation
+    output:
+      description: What clients send, by name, as the configuration documents it
+      type: object
+
+exposition:
+  isolated: true
+  /:
+    POST:
+      anonymous: true
+      map:ip: ip
+      map:country: country
+      map:headers:
+        origin: origin
+        agent: user-agent
+      io:throttle:
+        key: [ip]
+        requests: 60
+        interval: 60
+      io:status: status
+      io:output: [status, accepted]
+      help:method: null
+      endpoint: anonymous
+    GET:
+      auth:role: system:chronicles:encounters
+      mcp:tool: true
+      help:method:
+        title: What clients send
+        description: >
+          The encounters clients send, by name: what each tells, and what each field of its
+          payload is. It documents, and holds nothing back: a name it does not tell is written
+          as any other. A client sends a batch with `sent`, its clock as it sends, and each event
+          with `time`, its clock as the event happened — both read as they are, never corrected,
+          however wrong the clock is: what is kept is when the gateway received the batch, less
+          the distance between the two.
+      io:output: true
+      endpoint: describe
+  /:identity:
+    POST:
+      auth:id: identity
+      map:segments:
+        identity: identity
+      map:ip: ip
+      map:country: country
+      map:headers:
+        origin: origin
+        agent: user-agent
+      io:throttle:
+        key: [identity]
+        requests: 60
+        interval: 60
+      io:status: status
+      io:output: [status, accepted]
+      help:method: null
+      endpoint: identified

package/components/{chronicles.ingestion → chronicles.encounters}/operations/anonymous.js

@@ -1,4 +1,4 @@
 // what anyone sent, checked and written by the extension
 export async function effect(input, context) {
-  return await context.chronicles.ingest(input, null)
+  return await context.chronicles.write(input, null)
 }

package/components/chronicles.encounters/operations/describe.js

@@ -0,0 +1,4 @@
+// what the context documents of what clients send
+export function computation(_, context) {
+  return context.configuration
+}

package/components/{chronicles.ingestion → chronicles.encounters}/operations/identified.js

@@ -1,4 +1,4 @@
 // what a caller the gateway verified sent, written as that identity's
 export async function effect(input, context) {
-  return await context.chronicles.ingest(input, input.identity)
+  return await context.chronicles.write(input, input.identity)
 }

package/components/{chronicles.ingestion → chronicles.encounters}/types/toa.d.ts

@@ -2,6 +2,9 @@
 // What a manifest does not state belongs in a file of your own.
 
 import type { CodedError } from '@toa.io/core/types'
+import type { Chronicles } from '@toa.io/extensions.chronicles'
+import type { FetchInit } from '@toa.io/extensions.fetch'
+import type { Logs, Metrics, Span } from '@toa.io/extensions.telemetry'
 
 export type AnonymousInput = Record<string, unknown>
 
@@ -17,7 +20,17 @@ export type IdentifiedOutput = {
   accepted?: number
 }
 
+export type DescribeOutput = Record<string, unknown>
+
 export interface Component {
   anonymous: (request: { input: AnonymousInput, task?: boolean }) => Promise<AnonymousOutput | CodedError<"INVALID" | "OVERSIZED">>
   identified: (request: { input: IdentifiedInput, task?: boolean }) => Promise<IdentifiedOutput | CodedError<"INVALID" | "OVERSIZED">>
+  describe: (request: { input?: null, task?: boolean }) => Promise<DescribeOutput>
 }
+
+export type Configuration = Record<string, {
+  /** What the client tells by sending it */
+  description: string
+  /** What each field of its payload is, by path */
+  fields?: Record<string, string>
+}>

package/components/chronicles.queries/manifest.toa.yaml

@@ -10,13 +10,15 @@ operations:
     output:
       type: object
       properties:
-        dialect:
-          description: What a query is written in
+        guide:
+          description: How to read the log, and how its store is read
           type: string
         labels:
           type: array
         states:
           type: array
+        encounters:
+          type: array
         views:
           type: array
   query:
@@ -25,7 +27,7 @@ operations:
       type: object
       properties:
         sql:
-          description: One statement over the log, in the dialect `describe` names
+          description: One statement over the log, as the guide `describe` answers says
           type: string
           minLength: 1
           maxLength: 65536
@@ -58,6 +60,16 @@ operations:
           description: Seconds between computations of a materialized view; absent, a plain view
           type: integer
           minimum: 3600
+        title:
+          description: What the view is, in a few words
+          type: string
+          minLength: 1
+          maxLength: 100
+        description:
+          description: What a row of the view is, and what each of its columns counts
+          type: string
+          minLength: 1
+          maxLength: 4096
       required: [name, sql]
     errors: [REFUSED, RESERVED, EXHAUSTED]
   drop:
@@ -78,21 +90,32 @@ exposition:
       help:method:
         title: What the log holds
         description: >
-          The dialect queries are written in; the labels of `events`, with how many rows each
-          has, the first and last time of them and the fields their payloads have; the
-          components with a state in `states`, with how many entities each has and the fields
-          of the state; and the views defined over the log.
+          A guide to reading the log and its store, to read before a query — what each table and
+          column means, and which time is which; the labels of `events` the components declare,
+          each with what it means, the view that reads it in types, how many rows it has, the
+          first and last time of them, and its fields with their types and what they mean; the
+          components with a state in `states`, likewise; the names of `encounters` clients sent,
+          with when the newest batch was received, the fields of their payloads as they were sent
+          and what the context documents of them; and the views defined over the log, with their
+          titles, descriptions and when a materialized one was last refreshed. `states` is each
+          entity as it stands now: what it was before is read from its events.
       io:output: true
       endpoint: describe
     POST:
       help:method:
         title: Query the log
         description: >
-          One statement, in the dialect `chronicles.queries.GET` names, over `events`, a row
-          for everything that happened (id, label, component, time, received, entity, version,
-          trace, payload), `states`, each entity as it stands (component, entity, version, time,
-          received, payload), and the views. At most 30 seconds and 10000 rows; at the rows,
-          `truncated` is true.
+          One statement, as the guide `chronicles.queries.GET` answers says, over `events`, a row
+          for everything a component did (id, label, component, time, entity, version, identity,
+          trace, payload), `states`, each entity as it stands (component, entity, version,
+          identity, time, payload), `encounters`, what clients sent (id, name, time, received,
+          visitor, session, identity, ip, country, origin, agent, browser, os, device, payload),
+          the view of each declared label and state, and the views. Every `time` is when it
+          happened, by the server: a change when the component made it, a fact when it was sent,
+          an encounter when the gateway received its batch less how long before sending the
+          client measured it happened — never the client's clock as it reads, nor when the store
+          wrote the row. An encounter's `received` is when the gateway received its batch. At
+          most 30 seconds, 10000 rows and 64 MiB; at the rows, `truncated` is true.
       io:output: true
       endpoint: query
   /views/:name:
@@ -102,9 +125,10 @@ exposition:
       help:method:
         title: Define a view
         description: >
-          A view over the log, by a statement a query could run. Without `refresh` it is computed
-          when it is read; with `refresh`, in seconds and at least 3600, it is computed on that
-          cycle and read as a table. Defining a name again replaces the view.
+          A view over the log, by a statement a query could run, with a title and a description
+          that tell whoever finds it what it counts. Without `refresh` it is computed when it is
+          read; with `refresh`, in seconds and at least 3600, it is computed on that cycle and
+          read as a table. Defining a name again replaces the view.
       help:route:
         name: The view's name
       endpoint: define

package/components/chronicles.queries/operations/define.js

@@ -1,3 +1,3 @@
 export async function effect(input, context) {
-  return await context.chronicles.define(input.name, input.sql, input.refresh)
+  return await context.chronicles.define(input)
 }

package/components/chronicles.queries/types/toa.d.ts

@@ -4,15 +4,16 @@
 import type { CodedError } from '@toa.io/core/types'
 
 export type DescribeOutput = {
-  /** What a query is written in */
-  dialect?: string
+  /** How to read the log, and how its store is read */
+  guide?: string
   labels?: unknown[]
   states?: unknown[]
+  encounters?: unknown[]
   views?: unknown[]
 }
 
 export type QueryInput = {
-  /** One statement over the log, in the dialect `describe` names */
+  /** One statement over the log, as the guide `describe` answers says */
   sql: string
 }
 
@@ -29,6 +30,10 @@ export type DefineInput = {
   sql: string
   /** Seconds between computations of a materialized view; absent, a plain view */
   refresh?: number
+  /** What the view is, in a few words */
+  title?: string
+  /** What a row of the view is, and what each of its columns counts */
+  description?: string
 }
 
 export type DropInput = {

package/components/{chronicles.facts → chronicles.relay}/manifest.toa.yaml

@@ -1,14 +1,14 @@
 namespace: chronicles
-name: facts
+name: relay
 
-# The extension writes what this component is sent to the log.
+# The extension publishes what this component is tasked with to the broker of the log.
 chronicles: ~
 
 operations:
   record:
     type: effect
     input:
-      description: A fact a component sent, as the row it is written as
+      description: A fact a component sent, as the row it is published as
       type: object
       properties:
         id:
@@ -30,3 +30,14 @@ operations:
         payload:
           type: object
       required: [id, label, component, time, payload]
+  encounter:
+    type: effect
+    input:
+      description: The encounters of a batch a client sent, as the rows they are published as
+      type: object
+      properties:
+        encounters:
+          type: array
+          items:
+            type: object
+      required: [encounters]

package/components/chronicles.relay/operations/encounter.js

@@ -0,0 +1,4 @@
+// the encounters of a batch a client sent, published by the extension
+export async function effect(input, context) {
+  await context.chronicles.encounter(input.encounters)
+}

package/components/{chronicles.facts → chronicles.relay}/operations/record.js

@@ -1,4 +1,4 @@
-// a fact a component sent with `context.chronicles.send`, written by the extension
+// a fact a component sent with `context.chronicles.send`, published by the extension
 export async function effect(input, context) {
   await context.chronicles.record(input)
 }

package/components/{chronicles.facts → chronicles.relay}/types/toa.d.ts

@@ -14,6 +14,11 @@ export type RecordInput = {
   payload: Record<string, unknown>
 }
 
+export type EncounterInput = {
+  encounters: Array<Record<string, unknown>>
+}
+
 export interface Component {
   record: (request: { input: RecordInput, task?: boolean }) => Promise<unknown>
+  encounter: (request: { input: EncounterInput, task?: boolean }) => Promise<unknown>
 }

package/guides/clickhouse.md

@@ -0,0 +1,19 @@
+# ClickHouse
+
+Queries are ClickHouse SQL, one statement, without `FORMAT`.
+
+- Read `events`, `states` and `encounters` with `FINAL`: a row delivered twice is two until
+  ClickHouse merges them.
+- `payload` is a `JSON` column. A path is `payload.total`, `payload.shipping.country`; its type is
+  whatever was written, so cast it: `payload.total::Int64`, `toString(payload.path)`.
+  `JSONAllPaths(payload)` lists the paths of a row, `JSONAllPathsWithTypes(payload)` with types.
+- A declared field's `read` gives it in its type: an object as a named `Tuple`, read as
+  `shipping.country`, an array of objects as an `Array(Tuple(...))`, read with
+  `ARRAY JOIN items AS item` and `item.sku`. Every one is `Nullable`, and `NULL` where a row's
+  value was not of its type: `sum`, `avg` and `count(field)` skip `NULL`, `arraySum` refuses a
+  `Nullable` array — use `ARRAY JOIN` instead.
+- `time`, and an encounter's `received`, are `DateTime64(3, 'UTC')`: `toDate(time)`, `toStartOfHour(time)`,
+  `time >= now() - INTERVAL 7 DAY`.
+- A query runs at most 30 seconds and answers at most 10000 rows and 64 MiB; past the rows it is
+  `truncated`. Aggregate rather than read rows.
+- A view with `refresh` is computed on that cycle: `refreshed` says when it last was.

package/guides/log.md

@@ -0,0 +1,52 @@
+# Reading the log
+
+The log is three tables: `events`, `states` and `encounters`. Every row has `scope`, the
+application it is of; you read the rows of yours.
+
+- `events`: a row for everything a component did. `label` is `<component>.<event>` for a change,
+  with the `entity` that changed and its `version`, or `<component>.<name>` for a fact, with
+  neither. `identity` is whose the row is, where the component says; `trace` is the call that
+  caused it.
+- `states`: each entity as it stands now, by `component` and `entity`, at its latest `version`.
+  Nothing before it is kept: what an entity was at a time in the past is read from its events.
+- `encounters`: what clients sent, by `name`. `visitor` is the client's id of itself, kept across
+  sessions; `session` is one period of its activity; `identity` is set where a signed-in client
+  sent it. `ip` is the client's network, `country` its country, `origin` the site it was sent from,
+  `agent` its `user-agent`, and `browser`, `os` and `device` what the agent says it is.
+
+## Time
+
+Every row has `time`, when it happened, on the server's clock. No column is when the store wrote a
+row.
+
+- A change, in `events` and `states`: when the component made it — the entity's `UPDATED`, or the
+  property the component names, or else the moment of the change.
+- A fact: when the operation sent it.
+- An encounter: when the gateway received its batch, less how long before sending the batch the
+  client says the event happened. A client's clock may be off by minutes or years; what it
+  measures between two of its own moments is not. So `time` is the gateway's clock at the
+  distance the client measured: the order and the intervals of one client's encounters are exact,
+  and so is when they happened, to the time a request takes to arrive. A batch a client held
+  while offline is written at when its events happened.
+
+An encounter also has `received`, when the gateway received its batch: one `received` is shared by
+every encounter of a batch, and a `received` far behind now means clients stopped sending.
+
+Rows can arrive late — after the service that writes them was away — and are written at their own
+time, so a count of the last hours may still grow. Compare `time` of encounters and of events
+directly: both are the server's.
+
+A declared label's payload holds only what is declared. `chronicles.queries.GET` tells each
+declared field's type, what it means, and `read`: the expression that selects it from the table
+in its type — use it rather than casting the payload yourself. An encounter's
+payload is what the client sent, and its fields differ between versions of a client: two types of
+one field are two versions.
+
+A visitor is not a person: one identity may have many visitors, and a browser that clears its
+storage is a new one. The visitors of an identity are the rows that have both. Leave out
+`device = 'bot'` from any question of who visits. Safari on an iPad reports a desktop Mac.
+
+Nothing arrives in order: order by `time`, or by `version` within one entity. A view another agent
+defined says what it counts in its title and description; read them before reusing one, and give
+your own views both. A view is read by its name, which `chronicles.queries.GET` tells with its
+database.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@toa.io/extensions.chronicles",
-  "version": "1.0.0-alpha.332",
+  "version": "1.0.0-alpha.333",
   "type": "module",
   "description": "Toa Chronicles: a log of a context, kept in ClickHouse",
   "author": "temich <tema.gurtovoy@gmail.com>",
@@ -20,6 +20,8 @@
   "files": [
     "transpiled",
     "components",
+    "guides",
+    "store",
     "readme.md",
     "!**/tsconfig.tsbuildinfo",
     "!**/*.test.*",
@@ -29,9 +31,11 @@
     "transpile": "tsc"
   },
   "dependencies": {
-    "@toa.io/core": "1.0.0-alpha.332",
-    "@toa.io/definitions": "1.0.0-alpha.332",
+    "@toa.io/core": "1.0.0-alpha.333",
+    "@toa.io/definitions": "1.0.0-alpha.333",
     "@toa.io/generic": "1.0.0-alpha.332",
+    "comq": "0.28.0",
     "openspan": "1.0.0-alpha.330"
-  }
+  },
+  "gitHead": "ddb74b2f99026fae540a8a8e0a49c26cbd22886a"
 }