@oneuptime/common 12.0.7 → 12.0.8

package/Types/Telemetry/EntitySource.ts

@@ -0,0 +1,40 @@
+/*
+ * How a `TelemetryEntity` registry row came to exist. The registry started
+ * as a forward-only catalog populated purely from OTLP resource attributes;
+ * two later sources join it, and the three differ in exactly one way that
+ * matters — who owns the row's lifecycle.
+ *
+ * That ownership is what `PruneStaleEntities` keys on. It reaps rows whose
+ * `lastSeenAt` has aged past their type's TTL, which is only a correct
+ * staleness signal when something is actively re-bumping `lastSeenAt`. Two
+ * of the three sources have nothing doing that, so the prune sweep is
+ * scoped to `Discovered` alone — see the guard there.
+ */
+enum EntitySource {
+  /**
+   * Derived from telemetry resource attributes at ingest and re-bumped on
+   * every reconcile window for as long as the entity keeps emitting. Silence
+   * past the type's TTL genuinely means gone, so these are TTL-pruned.
+   */
+  Discovered = "discovered",
+
+  /**
+   * Mirrored from a OneUptime inventory table (NetworkDevice, CloudResource,
+   * IoTDevice, ...). These are collected by pollers rather than OTLP, so no
+   * ingest path bumps `lastSeenAt` — the owning row IS the liveness signal,
+   * and the mirror is kept in step by that table's create/delete hooks. TTL
+   * pruning one of these would delete the registry row for a device that is
+   * still very much registered.
+   */
+  Inventory = "inventory",
+
+  /**
+   * Created by a user for something that emits no telemetry at all — a
+   * third-party API, a vendor-managed database, an appliance. Nothing will
+   * ever bump `lastSeenAt`, so these are never TTL-pruned; they live until
+   * the user deletes them.
+   */
+  Manual = "manual",
+}
+
+export default EntitySource;

package/Types/Telemetry/EntityType.ts

@@ -51,6 +51,33 @@ enum EntityType {
   DockerSwarmService = "docker.swarm.service",
   DockerSwarmTask = "docker.swarm.task",
   TelemetrySdk = "telemetry.sdk",
+  /*
+   * Inventory-mirrored types. Unlike everything above, these are never
+   * derived from an OTLP resource — the estate they describe is collected
+   * by pollers (SNMP, cloud APIs, MQTT) into rich typed tables, and the
+   * registry row is a mirror of that row so the entity explorer can show
+   * one estate instead of one telemetry pipeline. Their identity is the
+   * owning row's own project-unique key, and their lifecycle follows it
+   * (see InventoryEntityRegistry); they carry EntitySource.Inventory and
+   * are deliberately absent from the prune TTL map.
+   */
+  NetworkDevice = "network.device",
+  CloudResource = "cloud.resource",
+  ServerlessFunction = "serverless.function",
+  RumApplication = "rum.application",
+  IoTDevice = "iot.device",
+  DockerHost = "docker.host",
+  PodmanHost = "podman.host",
+  /*
+   * Manual (user-authored) types, for the parts of an estate that emit no
+   * telemetry and appear in no inventory table: a third-party payment API,
+   * a vendor-managed database, a physical appliance. They exist so a
+   * dependency edge can terminate somewhere real instead of dangling at
+   * the boundary of what OneUptime can see. Carry EntitySource.Manual.
+   */
+  ExternalService = "external.service",
+  ExternalDatabase = "external.database",
+  Appliance = "appliance",
 }
 
 export default EntityType;

package/Types/Telemetry/EntityTypeGroups.ts

@@ -0,0 +1,65 @@
+import EntityType from "./EntityType";
+
+/*
+ * Partitions of the EntityType vocabulary by who creates and owns a row of
+ * that type. Isomorphic (plain Types, no server imports) because all three
+ * consumers need the same answer: the service enforcing what a user may
+ * create, the dashboard populating the type dropdown, and the tests pinning
+ * that the prune sweep cannot reach a type nothing re-bumps.
+ *
+ * Deliberately expressed as explicit sets rather than a string-prefix test.
+ * Prefix matching would silently absorb any future `external.*` or
+ * `*.device` type into the set that is exempt from pruning, which is the
+ * kind of default that goes unnoticed until a registry fills with rows
+ * nothing can ever reap.
+ */
+
+/**
+ * Types a user may create by hand. These describe things OneUptime cannot
+ * observe — a third-party API, a vendor-managed database, an appliance —
+ * so nothing will ever discover them and nothing will ever bump their
+ * `lastSeenAt`.
+ *
+ * Manual creation is confined to these on purpose. A hand-made row of an
+ * observable type (say `service`) would mint an entity key from the manual
+ * identity attribute, which by construction differs from the key ingest
+ * derives from `service.name` — so the manual row would sit beside the
+ * discovered one as a permanent near-duplicate rather than merging with it.
+ */
+export const MANUAL_ENTITY_TYPES: ReadonlySet<EntityType> = new Set<EntityType>(
+  [
+    EntityType.ExternalService,
+    EntityType.ExternalDatabase,
+    EntityType.Appliance,
+  ],
+);
+
+/**
+ * Types mirrored from a OneUptime inventory table. The owning row is the
+ * source of truth for both identity and liveness; the registry row is a
+ * projection kept in step by that table's create/delete hooks.
+ */
+export const INVENTORY_ENTITY_TYPES: ReadonlySet<EntityType> =
+  new Set<EntityType>([
+    EntityType.NetworkDevice,
+    EntityType.CloudResource,
+    EntityType.ServerlessFunction,
+    EntityType.RumApplication,
+    EntityType.IoTDevice,
+    EntityType.DockerHost,
+    EntityType.PodmanHost,
+  ]);
+
+/**
+ * True for types whose rows have no telemetry heartbeat, and which are
+ * therefore never eligible for `lastSeenAt`-based pruning. The prune job
+ * enforces this through the `source` column instead (a row's source is the
+ * authority, not its type); this predicate is what the tests assert the TTL
+ * map against, so a newly added type cannot quietly acquire a TTL.
+ */
+export function isNonTelemetryEntityType(entityType: EntityType): boolean {
+  return (
+    MANUAL_ENTITY_TYPES.has(entityType) ||
+    INVENTORY_ENTITY_TYPES.has(entityType)
+  );
+}

package/Types/Workflow/Component.ts

@@ -30,6 +30,35 @@ export enum ComponentInputType {
   WorkflowSelect = "Workflow Select",
 }
 
+/*
+ * Argument types whose value is a JSON document but is read with
+ * JSONFunctions.parse — that is, JSON5 — rather than JSON.parse. Unquoted keys,
+ * single quotes and trailing commas therefore work in these, and a workflow
+ * written that way runs fine today.
+ *
+ * The distinction matters wherever a JSON document is checked for syntax
+ * before it runs: applying JSON.parse's rules to one of these would report a
+ * mistake in a workflow that does not have one. The strict set is everything
+ * else — RunWorkflow.getComponentArguments calls JSON.parse directly on
+ * JSON, Query and Select, so those really must be JSON.
+ */
+const JSON5_TOLERANT_INPUT_TYPES: Array<ComponentInputType> = [
+  ComponentInputType.StringDictionary,
+  ComponentInputType.JSONArray,
+  ComponentInputType.BaseModel,
+  ComponentInputType.BaseModelArray,
+];
+
+export type IsJSON5ToleratedInputTypeFunction = (
+  type: ComponentInputType,
+) => boolean;
+
+export const isJSON5ToleratedInputType: IsJSON5ToleratedInputTypeFunction = (
+  type: ComponentInputType,
+): boolean => {
+  return JSON5_TOLERANT_INPUT_TYPES.includes(type);
+};
+
 export enum ComponentType {
   Trigger = "Trigger",
   Component = "Component",

package/Types/Workflow/Components/API.ts

@@ -37,6 +37,13 @@ const components: Array<ComponentMetadata> = [
         type: ComponentInputType.StringDictionary,
         required: false,
         isAdvanced: true,
+        /*
+         * Headers are where an Authorization bearer token gets typed, and
+         * without this the resolved value is written verbatim into the
+         * WorkflowLog for anyone with read access to the project. Only values
+         * that came from a variable marked secret were ever scrubbed.
+         */
+        isSensitive: true,
         placeholder:
           'Example: {"header1": "value1", "header2": "value2", ....}',
       },
@@ -123,6 +130,13 @@ const components: Array<ComponentMetadata> = [
         type: ComponentInputType.StringDictionary,
         required: false,
         isAdvanced: true,
+        /*
+         * Headers are where an Authorization bearer token gets typed, and
+         * without this the resolved value is written verbatim into the
+         * WorkflowLog for anyone with read access to the project. Only values
+         * that came from a variable marked secret were ever scrubbed.
+         */
+        isSensitive: true,
         placeholder:
           'Example: {"header1": "value1", "header2": "value2", ....}',
       },
@@ -209,6 +223,13 @@ const components: Array<ComponentMetadata> = [
         type: ComponentInputType.StringDictionary,
         required: false,
         isAdvanced: true,
+        /*
+         * Headers are where an Authorization bearer token gets typed, and
+         * without this the resolved value is written verbatim into the
+         * WorkflowLog for anyone with read access to the project. Only values
+         * that came from a variable marked secret were ever scrubbed.
+         */
+        isSensitive: true,
         placeholder:
           'Example: {"header1": "value1", "header2": "value2", ....}',
       },
@@ -295,6 +316,13 @@ const components: Array<ComponentMetadata> = [
         type: ComponentInputType.StringDictionary,
         required: false,
         isAdvanced: true,
+        /*
+         * Headers are where an Authorization bearer token gets typed, and
+         * without this the resolved value is written verbatim into the
+         * WorkflowLog for anyone with read access to the project. Only values
+         * that came from a variable marked secret were ever scrubbed.
+         */
+        isSensitive: true,
         placeholder:
           'Example: {"header1": "value1", "header2": "value2", ....}',
       },
@@ -381,6 +409,13 @@ const components: Array<ComponentMetadata> = [
         type: ComponentInputType.StringDictionary,
         required: false,
         isAdvanced: true,
+        /*
+         * Headers are where an Authorization bearer token gets typed, and
+         * without this the resolved value is written verbatim into the
+         * WorkflowLog for anyone with read access to the project. Only values
+         * that came from a variable marked secret were ever scrubbed.
+         */
+        isSensitive: true,
         placeholder:
           'Example: {"header1": "value1", "header2": "value2", ....}',
       },

package/Types/Workflow/Components/BaseModel.ts

@@ -19,9 +19,13 @@ export default class BaseModelComponent {
      * Every one of these arguments is keyed on column names, and the single
      * most common way to get one wrong is to write "id" - which is what an ID
      * is called everywhere else in the product - when the column is "_id".
-     * Both spellings work now, but the hint has to say so where the builder is
-     * actually typing, because the Query argument is a bare JSON editor with
-     * no field picker behind it.
+     * Both spellings work.
+     *
+     * Query and record arguments now get a row editor backed by the model's
+     * real column list (ModelColumnEditor), so the names are offered rather
+     * than remembered. The hint stays because that editor keeps a JSON escape
+     * hatch for the queries rows cannot express, and because it is still the
+     * only thing explaining the two spellings.
      */
     const columnHint: string = `Keys are column names on ${model.singularName}. The ID column is "_id" (the "id" spelling is accepted too).`;
 

package/Types/Workflow/StepTrace.ts

@@ -0,0 +1,181 @@
+/*
+ * A structured record of what each step of a workflow run actually did.
+ *
+ * The runner has always known this — it logs the arguments it resolved, the
+ * values a component returned, and which out port it took — but it flattened
+ * all of it into one newline-joined string, so the only way to answer "what did
+ * step three receive" was to read the whole blob. Keeping the same information
+ * as data lets the run be read as a list of steps, and lets a step be matched
+ * back to the node it came from on the canvas.
+ *
+ * The shapes live in Common/Types because the runner writes them, the API
+ * returns them, and the dashboard renders them.
+ */
+
+import { JSONObject, JSONValue } from "../JSON";
+
+export enum WorkflowStepStatus {
+  Success = "Success",
+  Error = "Error",
+}
+
+export interface WorkflowStepTraceEntry {
+  /** The step's user-facing id ("api-get-1") — matches a node on the canvas. */
+  componentId: string;
+  /** Which kind of component this was, e.g. "api-post". */
+  metadataId: string;
+  /** The component's title, so the trace reads without loading metadata. */
+  title: string;
+  status: WorkflowStepStatus;
+  startedAt: string;
+  completedAt: string;
+  durationInMs: number;
+  /** Resolved arguments, already redacted for sensitive fields. */
+  argumentValues: JSONObject;
+  /** Returned values, already redacted for sensitive fields. */
+  returnValues: JSONObject;
+  /** The out port taken, or null when the step ended the run. */
+  executedPort: string | null;
+  /** Present only on a failed step. */
+  errorMessage?: string | undefined;
+}
+
+export interface WorkflowStepTrace {
+  steps: Array<WorkflowStepTraceEntry>;
+  /**
+   * Set when steps were dropped to keep the row bounded. The dashboard says so
+   * rather than quietly showing a partial run as if it were complete.
+   */
+  truncated?: boolean | undefined;
+}
+
+/*
+ * A run is bounded by cycle detection, but a workflow that suspends and resumes
+ * accumulates across resumes, and a single value can be an entire HTTP response
+ * body. Both are capped: the row is written on every status change, and an
+ * unbounded trace would turn each of those writes into a large one.
+ */
+export const MAX_TRACE_STEPS: number = 100;
+export const MAX_TRACE_VALUE_LENGTH: number = 4000;
+
+export const TRUNCATED_VALUE_SUFFIX: string = "… (truncated)";
+
+export type TruncateTraceValueFunction = (value: JSONValue) => JSONValue;
+
+/**
+ * Shrink one recorded value to something a row can hold.
+ *
+ * Strings are cut; anything structured is measured by its JSON length and
+ * replaced wholesale when it is too big, because cutting a serialized object in
+ * half produces text that looks like data but cannot be parsed.
+ */
+export const truncateTraceValue: TruncateTraceValueFunction = (
+  value: JSONValue,
+): JSONValue => {
+  if (typeof value === "string") {
+    if (value.length <= MAX_TRACE_VALUE_LENGTH) {
+      return value;
+    }
+
+    return value.slice(0, MAX_TRACE_VALUE_LENGTH) + TRUNCATED_VALUE_SUFFIX;
+  }
+
+  if (value === null || value === undefined) {
+    return value;
+  }
+
+  if (typeof value !== "object") {
+    return value;
+  }
+
+  let serialized: string;
+
+  try {
+    serialized = JSON.stringify(value);
+  } catch {
+    // Circular or otherwise unserializable: say so rather than throwing.
+    return "[value could not be recorded]";
+  }
+
+  if (serialized.length <= MAX_TRACE_VALUE_LENGTH) {
+    return value;
+  }
+
+  return serialized.slice(0, MAX_TRACE_VALUE_LENGTH) + TRUNCATED_VALUE_SUFFIX;
+};
+
+export type TruncateTraceValuesFunction = (values: JSONObject) => JSONObject;
+
+export const truncateTraceValues: TruncateTraceValuesFunction = (
+  values: JSONObject,
+): JSONObject => {
+  const truncated: JSONObject = {};
+
+  for (const key of Object.keys(values || {})) {
+    truncated[key] = truncateTraceValue(values[key] as JSONValue);
+  }
+
+  return truncated;
+};
+
+export type AppendTraceStepFunction = (
+  trace: WorkflowStepTrace,
+  entry: WorkflowStepTraceEntry,
+) => WorkflowStepTrace;
+
+/**
+ * Add a step, keeping the trace within its cap.
+ *
+ * When the cap is reached the OLDEST steps are dropped: a run that failed is
+ * read from the end, and the step that broke it is the last one.
+ */
+export const appendTraceStep: AppendTraceStepFunction = (
+  trace: WorkflowStepTrace,
+  entry: WorkflowStepTraceEntry,
+): WorkflowStepTrace => {
+  const steps: Array<WorkflowStepTraceEntry> = [...(trace.steps || []), entry];
+
+  if (steps.length <= MAX_TRACE_STEPS) {
+    return { ...trace, steps: steps };
+  }
+
+  return {
+    ...trace,
+    steps: steps.slice(steps.length - MAX_TRACE_STEPS),
+    truncated: true,
+  };
+};
+
+export type EmptyTraceFunction = () => WorkflowStepTrace;
+
+export const emptyTrace: EmptyTraceFunction = (): WorkflowStepTrace => {
+  return { steps: [] };
+};
+
+export type ParseTraceFunction = (value: JSONValue) => WorkflowStepTrace;
+
+/**
+ * Read a trace back off a row, tolerating anything that is not one.
+ *
+ * Old runs predate the column and rows can be written by an older build, so
+ * this never throws — a trace it cannot understand reads as an empty one and
+ * the viewer falls back to the raw log.
+ */
+export const parseTrace: ParseTraceFunction = (
+  value: JSONValue,
+): WorkflowStepTrace => {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    return emptyTrace();
+  }
+
+  const steps: JSONValue | undefined = (value as JSONObject)["steps"];
+
+  if (!Array.isArray(steps)) {
+    return emptyTrace();
+  }
+
+  return {
+    steps: steps as unknown as Array<WorkflowStepTraceEntry>,
+    truncated: Boolean((value as JSONObject)["truncated"]),
+  };
+};