@oneuptime/common 12.0.6 → 12.0.8

package/Types/Telemetry/EntityTypeGroups.ts

@@ -0,0 +1,65 @@
+import EntityType from "./EntityType";
+
+/*
+ * Partitions of the EntityType vocabulary by who creates and owns a row of
+ * that type. Isomorphic (plain Types, no server imports) because all three
+ * consumers need the same answer: the service enforcing what a user may
+ * create, the dashboard populating the type dropdown, and the tests pinning
+ * that the prune sweep cannot reach a type nothing re-bumps.
+ *
+ * Deliberately expressed as explicit sets rather than a string-prefix test.
+ * Prefix matching would silently absorb any future `external.*` or
+ * `*.device` type into the set that is exempt from pruning, which is the
+ * kind of default that goes unnoticed until a registry fills with rows
+ * nothing can ever reap.
+ */
+
+/**
+ * Types a user may create by hand. These describe things OneUptime cannot
+ * observe — a third-party API, a vendor-managed database, an appliance —
+ * so nothing will ever discover them and nothing will ever bump their
+ * `lastSeenAt`.
+ *
+ * Manual creation is confined to these on purpose. A hand-made row of an
+ * observable type (say `service`) would mint an entity key from the manual
+ * identity attribute, which by construction differs from the key ingest
+ * derives from `service.name` — so the manual row would sit beside the
+ * discovered one as a permanent near-duplicate rather than merging with it.
+ */
+export const MANUAL_ENTITY_TYPES: ReadonlySet<EntityType> = new Set<EntityType>(
+  [
+    EntityType.ExternalService,
+    EntityType.ExternalDatabase,
+    EntityType.Appliance,
+  ],
+);
+
+/**
+ * Types mirrored from a OneUptime inventory table. The owning row is the
+ * source of truth for both identity and liveness; the registry row is a
+ * projection kept in step by that table's create/delete hooks.
+ */
+export const INVENTORY_ENTITY_TYPES: ReadonlySet<EntityType> =
+  new Set<EntityType>([
+    EntityType.NetworkDevice,
+    EntityType.CloudResource,
+    EntityType.ServerlessFunction,
+    EntityType.RumApplication,
+    EntityType.IoTDevice,
+    EntityType.DockerHost,
+    EntityType.PodmanHost,
+  ]);
+
+/**
+ * True for types whose rows have no telemetry heartbeat, and which are
+ * therefore never eligible for `lastSeenAt`-based pruning. The prune job
+ * enforces this through the `source` column instead (a row's source is the
+ * authority, not its type); this predicate is what the tests assert the TTL
+ * map against, so a newly added type cannot quietly acquire a TTL.
+ */
+export function isNonTelemetryEntityType(entityType: EntityType): boolean {
+  return (
+    MANUAL_ENTITY_TYPES.has(entityType) ||
+    INVENTORY_ENTITY_TYPES.has(entityType)
+  );
+}

package/Types/Telemetry/TelemetrySavedViewState.ts

@@ -19,7 +19,15 @@ export interface TelemetrySavedViewTimeRange {
 export default interface TelemetrySavedViewState {
   // Submitted search string from the explorer search bar.
   search?: string | undefined;
-  // Active facet filters as [facetKey, value] tuples.
+  /*
+   * Active facet filters as [facetKey, value] tuples.
+   *
+   * Read these with readSavedViewFilters (Common/Utils/Telemetry/
+   * SavedViewFilters) rather than destructuring them. Rows written before
+   * JSONFunctions.serializeValue learned to preserve nested arrays hold
+   * { "0": facetKey, "1": value } instead, and destructuring one of those
+   * throws — which is what this type asserting otherwise once cost us.
+   */
   filters?: Array<[string, string]> | undefined;
   // Selected time range.
   timeRange?: TelemetrySavedViewTimeRange | undefined;

package/Types/Workflow/Component.ts

@@ -30,6 +30,35 @@ export enum ComponentInputType {
   WorkflowSelect = "Workflow Select",
 }
 
+/*
+ * Argument types whose value is a JSON document but is read with
+ * JSONFunctions.parse — that is, JSON5 — rather than JSON.parse. Unquoted keys,
+ * single quotes and trailing commas therefore work in these, and a workflow
+ * written that way runs fine today.
+ *
+ * The distinction matters wherever a JSON document is checked for syntax
+ * before it runs: applying JSON.parse's rules to one of these would report a
+ * mistake in a workflow that does not have one. The strict set is everything
+ * else — RunWorkflow.getComponentArguments calls JSON.parse directly on
+ * JSON, Query and Select, so those really must be JSON.
+ */
+const JSON5_TOLERANT_INPUT_TYPES: Array<ComponentInputType> = [
+  ComponentInputType.StringDictionary,
+  ComponentInputType.JSONArray,
+  ComponentInputType.BaseModel,
+  ComponentInputType.BaseModelArray,
+];
+
+export type IsJSON5ToleratedInputTypeFunction = (
+  type: ComponentInputType,
+) => boolean;
+
+export const isJSON5ToleratedInputType: IsJSON5ToleratedInputTypeFunction = (
+  type: ComponentInputType,
+): boolean => {
+  return JSON5_TOLERANT_INPUT_TYPES.includes(type);
+};
+
 export enum ComponentType {
   Trigger = "Trigger",
   Component = "Component",

package/Types/Workflow/Components/API.ts

@@ -37,6 +37,13 @@ const components: Array<ComponentMetadata> = [
         type: ComponentInputType.StringDictionary,
         required: false,
         isAdvanced: true,
+        /*
+         * Headers are where an Authorization bearer token gets typed, and
+         * without this the resolved value is written verbatim into the
+         * WorkflowLog for anyone with read access to the project. Only values
+         * that came from a variable marked secret were ever scrubbed.
+         */
+        isSensitive: true,
         placeholder:
           'Example: {"header1": "value1", "header2": "value2", ....}',
       },
@@ -123,6 +130,13 @@ const components: Array<ComponentMetadata> = [
         type: ComponentInputType.StringDictionary,
         required: false,
         isAdvanced: true,
+        /*
+         * Headers are where an Authorization bearer token gets typed, and
+         * without this the resolved value is written verbatim into the
+         * WorkflowLog for anyone with read access to the project. Only values
+         * that came from a variable marked secret were ever scrubbed.
+         */
+        isSensitive: true,
         placeholder:
           'Example: {"header1": "value1", "header2": "value2", ....}',
       },
@@ -209,6 +223,13 @@ const components: Array<ComponentMetadata> = [
         type: ComponentInputType.StringDictionary,
         required: false,
         isAdvanced: true,
+        /*
+         * Headers are where an Authorization bearer token gets typed, and
+         * without this the resolved value is written verbatim into the
+         * WorkflowLog for anyone with read access to the project. Only values
+         * that came from a variable marked secret were ever scrubbed.
+         */
+        isSensitive: true,
         placeholder:
           'Example: {"header1": "value1", "header2": "value2", ....}',
       },
@@ -295,6 +316,13 @@ const components: Array<ComponentMetadata> = [
         type: ComponentInputType.StringDictionary,
         required: false,
         isAdvanced: true,
+        /*
+         * Headers are where an Authorization bearer token gets typed, and
+         * without this the resolved value is written verbatim into the
+         * WorkflowLog for anyone with read access to the project. Only values
+         * that came from a variable marked secret were ever scrubbed.
+         */
+        isSensitive: true,
         placeholder:
           'Example: {"header1": "value1", "header2": "value2", ....}',
       },
@@ -381,6 +409,13 @@ const components: Array<ComponentMetadata> = [
         type: ComponentInputType.StringDictionary,
         required: false,
         isAdvanced: true,
+        /*
+         * Headers are where an Authorization bearer token gets typed, and
+         * without this the resolved value is written verbatim into the
+         * WorkflowLog for anyone with read access to the project. Only values
+         * that came from a variable marked secret were ever scrubbed.
+         */
+        isSensitive: true,
         placeholder:
           'Example: {"header1": "value1", "header2": "value2", ....}',
       },

package/Types/Workflow/Components/BaseModel.ts

@@ -1,4 +1,5 @@
 import BaseModel from "../../../Models/DatabaseModels/DatabaseBaseModel/DatabaseBaseModel";
+import Route from "../../API/Route";
 import IconProp from "../../Icon/IconProp";
 import Text from "../../Text";
 import ComponentMetadata, {
@@ -14,6 +15,33 @@ export default class BaseModelComponent {
       return [];
     }
 
+    /*
+     * Every one of these arguments is keyed on column names, and the single
+     * most common way to get one wrong is to write "id" - which is what an ID
+     * is called everywhere else in the product - when the column is "_id".
+     * Both spellings work.
+     *
+     * Query and record arguments now get a row editor backed by the model's
+     * real column list (ModelColumnEditor), so the names are offered rather
+     * than remembered. The hint stays because that editor keeps a JSON escape
+     * hatch for the queries rows cannot express, and because it is still the
+     * only thing explaining the two spellings.
+     */
+    const columnHint: string = `Keys are column names on ${model.singularName}. The ID column is "_id" (the "id" spelling is accepted too).`;
+
+    const queryDescription: string = `Query that selects which ${model.pluralName} this component acts on. ${columnHint}`;
+
+    const queryPlaceholder: string =
+      'Example: {"_id": "00000000-0000-0000-0000-000000000000"}';
+
+    const selectPlaceholder: string = 'Example: {"_id": true, "name": true}';
+
+    const dataDescription: string = `${model.singularName} fields to write, as a JSON object. ${columnHint}`;
+
+    const documentationLink: Route = Route.fromString(
+      "/workflow/docs/DatabaseComponents.md",
+    );
+
     if (model.enableWorkflowOn.read) {
       components.push({
         id: `${Text.pascalCaseToDashes(model.tableName!)}-find-one`,
@@ -23,22 +51,23 @@ export default class BaseModelComponent {
         iconProp: IconProp.ArrowCircleDown,
         tableName: model.tableName!,
         componentType: ComponentType.Component,
+        documentationLink: documentationLink,
         arguments: [
           {
             type: ComponentInputType.Query,
             name: "Query",
-            description: `Query on ${model.singularName}`,
+            description: queryDescription,
             required: true,
             id: "query",
-            placeholder: 'Example: {"columnName": "value", ...}',
+            placeholder: queryPlaceholder,
           },
           {
             type: ComponentInputType.Select,
             name: "Select Fields",
-            description: `Select on ${model.singularName}`,
+            description: `Fields to read from ${model.singularName}. ${columnHint}`,
             required: true,
             id: "select",
-            placeholder: 'Example: {"columnName": true, ...}',
+            placeholder: selectPlaceholder,
           },
         ],
         returnValues: [
@@ -82,22 +111,23 @@ export default class BaseModelComponent {
         iconProp: IconProp.ArrowCircleDown,
         tableName: model.tableName!,
         componentType: ComponentType.Component,
+        documentationLink: documentationLink,
         arguments: [
           {
             type: ComponentInputType.Query,
             name: "Query",
-            description: "Please fill out this query",
+            description: queryDescription,
             required: true,
             id: "query",
-            placeholder: 'Example: {"columnName": "value", ...}',
+            placeholder: queryPlaceholder,
           },
           {
             type: ComponentInputType.Select,
             name: "Select Fields",
-            description: `Select on ${model.singularName}`,
+            description: `Fields to read from ${model.singularName}. ${columnHint}`,
             required: true,
             id: "select",
-            placeholder: 'Example: {"columnName": true, ...}',
+            placeholder: selectPlaceholder,
           },
           {
             type: ComponentInputType.Number,
@@ -196,17 +226,26 @@ export default class BaseModelComponent {
         iconProp: IconProp.Trash,
         tableName: model.tableName!,
         componentType: ComponentType.Component,
+        documentationLink: documentationLink,
         arguments: [
           {
             type: ComponentInputType.Query,
             name: "Delete by",
-            description: "Please fill out this query",
+            description: queryDescription,
             required: true,
             id: "query",
-            placeholder: 'Example: {"columnName": "value", ...}',
+            placeholder: queryPlaceholder,
+          },
+        ],
+        returnValues: [
+          {
+            id: "items-deleted",
+            name: "Items Deleted",
+            description: `How many ${model.pluralName} the query matched and deleted. Zero is not an error - it means nothing matched.`,
+            type: ComponentInputType.Number,
+            required: false,
           },
         ],
-        returnValues: [],
         inPorts: [
           {
             title: "In",
@@ -239,14 +278,15 @@ export default class BaseModelComponent {
         iconProp: IconProp.Trash,
         tableName: model.tableName!,
         componentType: ComponentType.Component,
+        documentationLink: documentationLink,
         arguments: [
           {
             type: ComponentInputType.Query,
             name: "Delete by",
-            description: "Please fill out this query",
+            description: queryDescription,
             required: true,
             id: "query",
-            placeholder: 'Example: {"columnName": "value", ...}',
+            placeholder: queryPlaceholder,
           },
           {
             type: ComponentInputType.Number,
@@ -263,7 +303,15 @@ export default class BaseModelComponent {
             id: "limit",
           },
         ],
-        returnValues: [],
+        returnValues: [
+          {
+            id: "items-deleted",
+            name: "Items Deleted",
+            description: `How many ${model.pluralName} the query matched and deleted. Zero is not an error - it means nothing matched.`,
+            type: ComponentInputType.Number,
+            required: false,
+          },
+        ],
         inPorts: [
           {
             title: "In",
@@ -312,10 +360,10 @@ export default class BaseModelComponent {
           {
             type: ComponentInputType.Select,
             name: "Select Fields",
-            description: `Select on ${model.singularName}`,
+            description: `Fields to read from ${model.singularName}. ${columnHint}`,
             required: true,
             id: "select",
-            placeholder: 'Example: {"columnName": true, ...}',
+            placeholder: selectPlaceholder,
           },
         ],
         returnValues: [
@@ -346,12 +394,13 @@ export default class BaseModelComponent {
         iconProp: IconProp.Database,
         tableName: model.tableName!,
         componentType: ComponentType.Component,
+        documentationLink: documentationLink,
         arguments: [
           {
             id: "json",
             placeholder: 'Example: {"columnName": "value", ...}',
             name: "JSON Object",
-            description: `${model.singularName} represented as JSON`,
+            description: `${model.singularName} represented as JSON. ${columnHint}`,
             type: ComponentInputType.JSON,
             required: true,
           },
@@ -397,12 +446,13 @@ export default class BaseModelComponent {
         iconProp: IconProp.Database,
         tableName: model.tableName!,
         componentType: ComponentType.Component,
+        documentationLink: documentationLink,
         arguments: [
           {
             id: "json-array",
             name: "JSON Array",
             placeholder: 'Example: [{"columnName": "value", ...}, {...}]',
-            description: "List of models represented as JSON array",
+            description: `List of ${model.pluralName} represented as a JSON array. ${columnHint}`,
             type: ComponentInputType.JSONArray,
             required: true,
           },
@@ -472,10 +522,10 @@ export default class BaseModelComponent {
           {
             type: ComponentInputType.Select,
             name: "Select Fields",
-            description: `Select on ${model.singularName}`,
+            description: `Fields to read from ${model.singularName}. ${columnHint}`,
             required: true,
             id: "select",
-            placeholder: 'Example: {"columnName": true, ...}',
+            placeholder: selectPlaceholder,
           },
         ],
         returnValues: [
@@ -505,25 +555,34 @@ export default class BaseModelComponent {
         iconProp: IconProp.ArrowCircleUp,
         tableName: model.tableName!,
         componentType: ComponentType.Component,
+        documentationLink: documentationLink,
         arguments: [
           {
             type: ComponentInputType.Query,
             name: "Query",
-            description: "Please fill out this query",
+            description: queryDescription,
             required: true,
             id: "query",
-            placeholder: 'Example: {"columnName": "value", ...}',
+            placeholder: queryPlaceholder,
           },
           {
             id: "data",
             placeholder: 'Example: {"columnName": "value", ...}',
             name: "Data (JSON Object)",
-            description: `${model.singularName} represented as JSON`,
+            description: dataDescription,
             type: ComponentInputType.JSON,
             required: true,
           },
         ],
-        returnValues: [],
+        returnValues: [
+          {
+            id: "items-updated",
+            name: "Items Updated",
+            description: `How many ${model.pluralName} the query matched and updated. Zero is not an error - it means nothing matched.`,
+            type: ComponentInputType.Number,
+            required: false,
+          },
+        ],
         inPorts: [
           {
             title: "In",
@@ -556,20 +615,21 @@ export default class BaseModelComponent {
         iconProp: IconProp.ArrowCircleUp,
         tableName: model.tableName!,
         componentType: ComponentType.Component,
+        documentationLink: documentationLink,
         arguments: [
           {
             type: ComponentInputType.Query,
             name: "Query",
-            description: "Please fill out this query",
+            description: queryDescription,
             required: true,
             id: "query",
-            placeholder: 'Example: {"columnName": "value", ...}',
+            placeholder: queryPlaceholder,
           },
           {
             id: "data",
             name: "Data (JSON Object)",
             placeholder: 'Example: {"columnName": "value", ...}',
-            description: `${model.singularName} represented as JSON`,
+            description: dataDescription,
             type: ComponentInputType.JSON,
             required: true,
           },
@@ -588,7 +648,15 @@ export default class BaseModelComponent {
             id: "limit",
           },
         ],
-        returnValues: [],
+        returnValues: [
+          {
+            id: "items-updated",
+            name: "Items Updated",
+            description: `How many ${model.pluralName} the query matched and updated. Zero is not an error - it means nothing matched.`,
+            type: ComponentInputType.Number,
+            required: false,
+          },
+        ],
         inPorts: [
           {
             title: "In",

package/Types/Workflow/StepTrace.ts

@@ -0,0 +1,181 @@
+/*
+ * A structured record of what each step of a workflow run actually did.
+ *
+ * The runner has always known this — it logs the arguments it resolved, the
+ * values a component returned, and which out port it took — but it flattened
+ * all of it into one newline-joined string, so the only way to answer "what did
+ * step three receive" was to read the whole blob. Keeping the same information
+ * as data lets the run be read as a list of steps, and lets a step be matched
+ * back to the node it came from on the canvas.
+ *
+ * The shapes live in Common/Types because the runner writes them, the API
+ * returns them, and the dashboard renders them.
+ */
+
+import { JSONObject, JSONValue } from "../JSON";
+
+export enum WorkflowStepStatus {
+  Success = "Success",
+  Error = "Error",
+}
+
+export interface WorkflowStepTraceEntry {
+  /** The step's user-facing id ("api-get-1") — matches a node on the canvas. */
+  componentId: string;
+  /** Which kind of component this was, e.g. "api-post". */
+  metadataId: string;
+  /** The component's title, so the trace reads without loading metadata. */
+  title: string;
+  status: WorkflowStepStatus;
+  startedAt: string;
+  completedAt: string;
+  durationInMs: number;
+  /** Resolved arguments, already redacted for sensitive fields. */
+  argumentValues: JSONObject;
+  /** Returned values, already redacted for sensitive fields. */
+  returnValues: JSONObject;
+  /** The out port taken, or null when the step ended the run. */
+  executedPort: string | null;
+  /** Present only on a failed step. */
+  errorMessage?: string | undefined;
+}
+
+export interface WorkflowStepTrace {
+  steps: Array<WorkflowStepTraceEntry>;
+  /**
+   * Set when steps were dropped to keep the row bounded. The dashboard says so
+   * rather than quietly showing a partial run as if it were complete.
+   */
+  truncated?: boolean | undefined;
+}
+
+/*
+ * A run is bounded by cycle detection, but a workflow that suspends and resumes
+ * accumulates across resumes, and a single value can be an entire HTTP response
+ * body. Both are capped: the row is written on every status change, and an
+ * unbounded trace would turn each of those writes into a large one.
+ */
+export const MAX_TRACE_STEPS: number = 100;
+export const MAX_TRACE_VALUE_LENGTH: number = 4000;
+
+export const TRUNCATED_VALUE_SUFFIX: string = "… (truncated)";
+
+export type TruncateTraceValueFunction = (value: JSONValue) => JSONValue;
+
+/**
+ * Shrink one recorded value to something a row can hold.
+ *
+ * Strings are cut; anything structured is measured by its JSON length and
+ * replaced wholesale when it is too big, because cutting a serialized object in
+ * half produces text that looks like data but cannot be parsed.
+ */
+export const truncateTraceValue: TruncateTraceValueFunction = (
+  value: JSONValue,
+): JSONValue => {
+  if (typeof value === "string") {
+    if (value.length <= MAX_TRACE_VALUE_LENGTH) {
+      return value;
+    }
+
+    return value.slice(0, MAX_TRACE_VALUE_LENGTH) + TRUNCATED_VALUE_SUFFIX;
+  }
+
+  if (value === null || value === undefined) {
+    return value;
+  }
+
+  if (typeof value !== "object") {
+    return value;
+  }
+
+  let serialized: string;
+
+  try {
+    serialized = JSON.stringify(value);
+  } catch {
+    // Circular or otherwise unserializable: say so rather than throwing.
+    return "[value could not be recorded]";
+  }
+
+  if (serialized.length <= MAX_TRACE_VALUE_LENGTH) {
+    return value;
+  }
+
+  return serialized.slice(0, MAX_TRACE_VALUE_LENGTH) + TRUNCATED_VALUE_SUFFIX;
+};
+
+export type TruncateTraceValuesFunction = (values: JSONObject) => JSONObject;
+
+export const truncateTraceValues: TruncateTraceValuesFunction = (
+  values: JSONObject,
+): JSONObject => {
+  const truncated: JSONObject = {};
+
+  for (const key of Object.keys(values || {})) {
+    truncated[key] = truncateTraceValue(values[key] as JSONValue);
+  }
+
+  return truncated;
+};
+
+export type AppendTraceStepFunction = (
+  trace: WorkflowStepTrace,
+  entry: WorkflowStepTraceEntry,
+) => WorkflowStepTrace;
+
+/**
+ * Add a step, keeping the trace within its cap.
+ *
+ * When the cap is reached the OLDEST steps are dropped: a run that failed is
+ * read from the end, and the step that broke it is the last one.
+ */
+export const appendTraceStep: AppendTraceStepFunction = (
+  trace: WorkflowStepTrace,
+  entry: WorkflowStepTraceEntry,
+): WorkflowStepTrace => {
+  const steps: Array<WorkflowStepTraceEntry> = [...(trace.steps || []), entry];
+
+  if (steps.length <= MAX_TRACE_STEPS) {
+    return { ...trace, steps: steps };
+  }
+
+  return {
+    ...trace,
+    steps: steps.slice(steps.length - MAX_TRACE_STEPS),
+    truncated: true,
+  };
+};
+
+export type EmptyTraceFunction = () => WorkflowStepTrace;
+
+export const emptyTrace: EmptyTraceFunction = (): WorkflowStepTrace => {
+  return { steps: [] };
+};
+
+export type ParseTraceFunction = (value: JSONValue) => WorkflowStepTrace;
+
+/**
+ * Read a trace back off a row, tolerating anything that is not one.
+ *
+ * Old runs predate the column and rows can be written by an older build, so
+ * this never throws — a trace it cannot understand reads as an empty one and
+ * the viewer falls back to the raw log.
+ */
+export const parseTrace: ParseTraceFunction = (
+  value: JSONValue,
+): WorkflowStepTrace => {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    return emptyTrace();
+  }
+
+  const steps: JSONValue | undefined = (value as JSONObject)["steps"];
+
+  if (!Array.isArray(steps)) {
+    return emptyTrace();
+  }
+
+  return {
+    steps: steps as unknown as Array<WorkflowStepTraceEntry>,
+    truncated: Boolean((value as JSONObject)["truncated"]),
+  };
+};