@servicenow/sdk-build-core 4.2.0 → 4.4.0

package/src/plugins/plugin.ts

@@ -25,6 +25,11 @@ export type Result<Value = unknown> =
           success: true
           value: Value
       }
+    | {
+          success: 'partial'
+          value: Value
+          unhandledRecords: Record[]
+      }
 
 export type CommitResult = { success: boolean }
 
@@ -92,6 +97,36 @@ export type CoalesceStrategy =
 
 export type FileType = 'fluent' | 'module' | 'json' | 'unknown'
 
+export type PluginApiDoc = {
+    /**
+     * The name of the API (e.g., 'BusinessRule', 'Acl', 'Table')
+     * This is the callee name used in fluent code.
+     */
+    apiName: string
+
+    /**
+     * Uses module-specifier path to the markdown documentation file for this API relative to the SDK root.
+     * For example, @servicenow/sdk/docs/BusinessRule/BusinessRule.md
+     */
+    docPath: string
+
+    /**
+     * Tags for categorizing and organizing the API documentation
+     */
+    tags: string[]
+}
+
+/**
+ * Manifest structure for API documentation.
+ * This is the opinionated format returned by getDocsMetadata().
+ */
+export type DocsManifest = {
+    [apiName: string]: {
+        docPath: string
+        tags: string[]
+    }
+}
+
 export type PluginConfig<
     Nodes extends SupportedKindName[] = SupportedKindName[],
     Shapes extends ShapeClass[] = ShapeClass[],
@@ -103,6 +138,12 @@ export type PluginConfig<
      */
     name: `${string}Plugin`
 
+    /**
+     * Documentation for the APIs provided by this plugin. This is used to automatically
+     * generate documentation that can be retrieved via the getDocsMetadata() method.
+     */
+    docs?: PluginApiDoc[]
+
     /**
      * The TypeScript AST nodes this plugin handles. Plugins that do not introduce new
      * syntax should not need to define any handlers here.
@@ -380,7 +421,12 @@ export type PluginConfig<
              * @returns A result indicating whether diffing was successful and, if so, a database
              *          containing only the changed records
              */
-            diff?: (existing: Database, incoming: Database, context: Context) => Promise<Result<Database>>
+            diff?: (
+                existing: Database,
+                incoming: Database,
+                descendants: Database,
+                context: Context
+            ) => Promise<Result<Database>>
         }
     }
 
@@ -595,6 +641,28 @@ export class Plugin {
         return this.config.name
     }
 
+    /**
+     * Get documentation metadata for APIs provided by this plugin.
+     * Returns a manifest object with raw (unresolved) documentation paths.
+     * Path resolution is handled by PluginRegistry.getDocsMetadata() using the isomorphic resolver.
+     */
+    getDocsMetadata(): DocsManifest | undefined {
+        if (!this.config.docs) {
+            return undefined
+        }
+
+        const manifest: DocsManifest = {}
+
+        for (const doc of this.config.docs) {
+            manifest[doc.apiName] = {
+                docPath: doc.docPath,
+                tags: doc.tags,
+            }
+        }
+
+        return manifest
+    }
+
     getDescendants(parent: Record, database: Database): Record[] {
         const descendantRelationships = Object.entries(this.relationships[parent.getTable()] ?? {}).filter(
             ([, { descendant }]) => descendant
@@ -720,7 +788,7 @@ export class Plugin {
     async nodeToShape(node: SupportedNode, context: Omit<Context, 'self'>): Promise<Result<Shape>> {
         const entry = this.nodeToShapeCache.get(node.compilerNode)
         if (entry) {
-            return entry.isError() ? { success: false } : entry.unwrap()
+            return entry.success ? entry.value : { success: false }
         }
 
         try {
@@ -773,7 +841,7 @@ export class Plugin {
     async shapeToSubclass<const S extends Shape>(shape: S, context: Omit<Context, 'self'>): Promise<Result<S>> {
         const entry = this.shapeToSubclassCache.get(shape)
         if (entry) {
-            return entry.isError() ? { success: false } : (entry.unwrap() as Result<S>)
+            return entry.success ? (entry.value as Result<S>) : { success: false }
         }
 
         try {
@@ -813,7 +881,7 @@ export class Plugin {
     async shapeToRecord(shape: Shape, context: Omit<Context, 'self'>): Promise<Result<Record>> {
         const entry = this.shapeToRecordCache.get(shape)
         if (entry) {
-            return entry.isError() ? { success: false } : entry.unwrap()
+            return entry.success ? entry.value : { success: false }
         }
 
         try {
@@ -910,6 +978,8 @@ export class Plugin {
                     continue
                 }
 
+                context.logger.debug(`Database contains ${database.query().length} records for transformation.`)
+
                 context.logger.debug(
                     `Transforming record into shape: ${record.getTable()}.${record.getId().getValue()}`
                 )
@@ -943,12 +1013,34 @@ export class Plugin {
                     descendants: nonDeletedDescendants,
                 })
 
+                // Track unhandled records to avoid marking them as handled
+                const unhandledRecords: Record[] = []
+                const unhandledRecordIds: string[] = []
+
+                if (result.success === 'partial') {
+                    for (const unhandledRecord of result.unhandledRecords) {
+                        unhandledRecords.push(unhandledRecord)
+                        unhandledRecordIds.push(unhandledRecord.getId().getValue())
+                    }
+                }
+
                 if (result.success) {
                     success = true
                     handledRecords.insert(record)
                     shapes.push(...[result.value].flat())
+
+                    // Mark all descendants as handled
                     for (const descendant of [...deletedDescendants, ...nonDeletedDescendants]) {
-                        handledRecords.insert(descendant)
+                        if (!unhandledRecordIds.includes(descendant.getId().getValue())) {
+                            handledRecords.insert(descendant)
+                        }
+                    }
+                    context.logger.debug(
+                        `Plugin ${this.getName()} deferred handling of records: ${unhandledRecords.map((r) => r.getTable() + '_' + r.getId().getValue()).join(', ')}`
+                    )
+
+                    if (unhandledRecords.length > 0) {
+                        return { success: 'partial', value: shapes, unhandledRecords }
                     }
                 }
             }
@@ -998,10 +1090,15 @@ export class Plugin {
                     ...[incoming.resolve(record.getId())].filter((r) => r !== undefined),
                     ...incoming.query().filter((r) => descendantsDatabase.resolve(r.getId()) !== undefined),
                 ]
-                const result = await diff.bind(this)(new Database(existingTree), new Database(incomingTree), {
-                    ...context,
-                    self: this,
-                })
+                const result = await diff.bind(this)(
+                    new Database(existingTree),
+                    new Database(incomingTree),
+                    new Database(descendants),
+                    {
+                        ...context,
+                        self: this,
+                    }
+                )
 
                 if (result.success) {
                     changedRecords.push(...result.value.query())
@@ -1245,4 +1342,14 @@ export class Plugins {
 
         return false
     }
+
+    getDocsMetadata(): DocsManifest {
+        return this.plugins.reduce((aggregated, plugin) => {
+            const pluginManifest = plugin.getDocsMetadata()
+            if (pluginManifest) {
+                Object.assign(aggregated, pluginManifest)
+            }
+            return aggregated
+        }, {} as DocsManifest)
+    }
 }

package/src/plugins/shape.ts

@@ -366,6 +366,10 @@ export class UndefinedShape extends Shape<undefined> {
     override toString(): StringShape {
         return Shape.from(this, '').asString()
     }
+
+    override getCode(): string {
+        return 'undefined'
+    }
 }
 
 export class DeletedShape extends UndefinedShape {}
@@ -473,6 +477,25 @@ export class PropertyAccessShape extends ResolvableShape {
     override getCode(): string {
         return this.elements.map((e) => e.getCode()).join('.')
     }
+
+    /**
+     * Override equals to compare property access paths instead of resolved values.
+     * This is important for datapill references where we need to compare the actual
+     * property paths (e.g., params.trigger.table_name vs params.trigger.current.user_name)
+     * rather than their resolved values (which would both be UnresolvedShape).
+     */
+    override equals(other: unknown): boolean {
+        if (other instanceof PropertyAccessShape) {
+            // Compare the property access paths element by element
+            const otherElements = other.getElements()
+            if (this.elements.length !== otherElements.length) {
+                return false
+            }
+            return this.elements.every((el, i) => el.getName() === otherElements[i]?.getName())
+        }
+        // Fall back to default behavior for non-PropertyAccessShape comparisons
+        return super.equals(other)
+    }
 }
 
 type StringContentType = 'plain' | 'cdata'
@@ -800,7 +823,7 @@ const coerceTo = (coerceToType: 'string' | 'number' | 'boolean' | 'cdata', value
     }
 }
 
-class Transform {
+export class ShapeTransform {
     constructor(
         private readonly shape: ObjectShape,
         private readonly resolve = true,
@@ -844,35 +867,35 @@ class Transform {
     }
 
     from(...properties: [string, ...string[]]) {
-        return new Transform(this.shape, this.resolve, properties, this._map, this._def, this._coerce)
+        return new ShapeTransform(this.shape, this.resolve, properties, this._map, this._def, this._coerce)
     }
 
     map(mapFunction: (...from: Shape[]) => unknown) {
-        return new Transform(this.shape, this.resolve, this._from, mapFunction, this._def, this._coerce)
+        return new ShapeTransform(this.shape, this.resolve, this._from, mapFunction, this._def, this._coerce)
     }
 
     val(value: unknown) {
-        return new Transform(this.shape, this.resolve, this._from, () => value, this._def, this._coerce)
+        return new ShapeTransform(this.shape, this.resolve, this._from, () => value, this._def, this._coerce)
     }
 
     def(value: unknown) {
-        return new Transform(this.shape, this.resolve, this._from, this._map, value, this._coerce)
+        return new ShapeTransform(this.shape, this.resolve, this._from, this._map, value, this._coerce)
     }
 
     toCdata() {
-        return new Transform(this.shape, this.resolve, this._from, this._map, this._def, 'cdata')
+        return new ShapeTransform(this.shape, this.resolve, this._from, this._map, this._def, 'cdata')
     }
 
     toString() {
-        return new Transform(this.shape, this.resolve, this._from, this._map, this._def, 'string')
+        return new ShapeTransform(this.shape, this.resolve, this._from, this._map, this._def, 'string')
     }
 
     toNumber() {
-        return new Transform(this.shape, this.resolve, this._from, this._map, this._def, 'number')
+        return new ShapeTransform(this.shape, this.resolve, this._from, this._map, this._def, 'number')
     }
 
     toBoolean() {
-        return new Transform(this.shape, this.resolve, this._from, this._map, this._def, 'boolean')
+        return new ShapeTransform(this.shape, this.resolve, this._from, this._map, this._def, 'boolean')
     }
 }
 
@@ -936,26 +959,37 @@ export class ObjectShape extends Shape<globalThis.Record<string, unknown>> {
         nonDefaultsOnly = false,
         resolve = true,
     }: ObjectPropertyOptions = {}): globalThis.Record<string, Shape> {
+        const result: globalThis.Record<string, Shape> = {}
+
         if (nonDefaultsOnly) {
-            const entries = Object.entries(this.setProperties).filter(
-                ([k, v]) => v.isDefined() && !v.equals(this.getDefault(k))
-            )
+            // Only include properties that differ from defaults
+            for (const k in this.setProperties) {
+                const v = this.setProperties[k]
+                if (v && v.isDefined() && !v.equals(this.getDefault(k))) {
+                    result[k] = resolve && v.isResolvable() ? v.resolve() : v
+                }
+            }
+            return result
+        }
 
-            return Object.fromEntries(
-                resolve ? entries.map(([k, v]) => [k, v.isResolvable() ? v.resolve() : v]) : entries
-            )
+        // Iterate set properties first to preserve ordering
+        for (const k in this.setProperties) {
+            const v = this.setProperties[k]
+            if (v) {
+                result[k] = resolve && v.isResolvable() ? v.resolve() : v
+            }
         }
 
-        // This logic may seem odd, but it's written this way to preserve the ordering of
-        // the set properties instead of using the ordering of the defaults.
-        const entries = [
-            ...Object.entries(this.setProperties),
-            ...Object.entries(this.defaultProperties).filter(
-                ([k]) => !this.setProperties[k] || this.setProperties[k].isUndefined()
-            ),
-        ]
+        for (const k in this.defaultProperties) {
+            if (!this.setProperties[k] || this.setProperties[k].isUndefined()) {
+                const v = this.defaultProperties[k]
+                if (v) {
+                    result[k] = resolve && v.isResolvable() ? v.resolve() : v
+                }
+            }
+        }
 
-        return Object.fromEntries(resolve ? entries.map(([k, v]) => [k, v.isResolvable() ? v.resolve() : v]) : entries)
+        return result
     }
 
     override getValue(nonDefaultsOnly = false): globalThis.Record<string, unknown> {
@@ -966,8 +1000,8 @@ export class ObjectShape extends Shape<globalThis.Record<string, unknown>> {
         const [property, next, ...rest] = typeof propertyOrPath === 'string' ? [propertyOrPath] : propertyOrPath
         const propertyAliases = [property, ...this.getAliases(property)]
         const properties = this.properties({ resolve })
-        const match = propertyAliases.find((p) => properties[p])
-        const value = match ? properties[match] : undefined
+        const match = propertyAliases.find((p) => p in properties)
+        const value = match !== undefined ? properties[match] : undefined
         return (
             (next ? value?.ifObject()?.get([next, ...rest], resolve) : value) ??
             new UndefinedShape({ source: this.getSource() })
@@ -1035,15 +1069,15 @@ export class ObjectShape extends Shape<globalThis.Record<string, unknown>> {
     }
 
     transform(
-        schema: (utils: { $: Transform; merge: typeof MERGE }) => {
-            [P in string | typeof MERGE]?: Transform
+        schema: (utils: { $: ShapeTransform; merge: typeof MERGE }) => {
+            [P in string | typeof MERGE]?: ShapeTransform
         },
         resolve = true
     ) {
         const properties = {}
         const defaults = {}
 
-        const definedSchema = schema({ merge: MERGE, $: new Transform(this, resolve) })
+        const definedSchema = schema({ merge: MERGE, $: new ShapeTransform(this, resolve) })
         for (const to of [...Object.keys(definedSchema), MERGE] as const) {
             if (definedSchema[to]) {
                 const result = definedSchema[to]._(to)
@@ -1099,7 +1133,7 @@ export class ObjectShape extends Shape<globalThis.Record<string, unknown>> {
                 } else if (a[key]?.equals(value)) {
                     // Don't overwrite properties that are equal
                 } else if (a[key]?.isObject() && value.isObject()) {
-                    a[key].merge(value) // Merge objects recursively
+                    a[key] = a[key].merge(value) // Merge objects recursively
                 } else {
                     a[key] = value
                 }

package/src/taxonomy.ts

@@ -13,6 +13,31 @@ const TableNames = {
     SYSEVENT_EMAIL_ACTION: 'sysevent_email_action',
     SYS_PD_PROCESS_DEFINITION: 'sys_pd_process_definition',
     SYS_HUB_TRIGGER_TEMPLATE: 'sys_hub_trigger_template',
+    SYS_VARIABLE_VALUE: 'sys_variable_value',
+    SYS_ELEMENT_MAPPING: 'sys_element_mapping',
+    SYS_HUB_FLOW_INPUT: 'sys_hub_flow_input',
+    SYS_HUB_FLOW_OUTPUT: 'sys_hub_flow_output',
+    SYS_HUB_ACTION_INPUT_ACTION_INSTANCE: 'sys_hub_action_input_action_instance',
+    SYS_HUB_FLOW_VARIABLE: 'sys_hub_flow_variable',
+    SYS_HUB_ACTION_INSTANCE: 'sys_hub_action_instance',
+    SYS_HUB_SUB_FLOW_INSTANCE: 'sys_hub_sub_flow_instance',
+    SYS_HUB_TRIGGER_INSTANCE: 'sys_hub_trigger_instance',
+    SYS_HUB_FLOW_LOGIC: 'sys_hub_flow_logic',
+    SYS_HUB_PILL_COMPOUND: 'sys_hub_pill_compound',
+    SYS_FLOW_CAT_VARIABLE_MODEL: 'sys_flow_cat_variable_model',
+    SYS_HUB_TRIGGER_INSTANCE_V2: 'sys_hub_trigger_instance_v2',
+    SYS_HUB_ACTION_INSTANCE_V2: 'sys_hub_action_instance_v2',
+    SYS_HUB_FLOW_LOGIC_INSTANCE_V2: 'sys_hub_flow_logic_instance_v2',
+    SYS_HUB_SUB_FLOW_INSTANCE_V2: 'sys_hub_sub_flow_instance_v2',
+    SYS_FLOW_RECORD_TRIGGER: 'sys_flow_record_trigger',
+    SYS_TRIGGER_RUNNER_MAPPING: 'sys_trigger_runner_mapping',
+    SYS_FLOW_TRIGGER_PLAN: 'sys_flow_trigger_plan',
+    SYS_HUB_FLOW_SNAPSHOT: 'sys_hub_flow_snapshot',
+    SYS_FLOW_SUBFLOW_PLAN: 'sys_flow_subflow_plan',
+    SYS_FLOW_EMAIL_TRIGGER: 'sys_flow_email_trigger',
+    SYS_FLOW_KNOWLEDGE_TRIGGER: 'sys_flow_knowledge_trigger',
+    SYS_FLOW_RT_QUERY_TRIGGER: 'sys_flow_rt_query_trigger',
+    SYS_FLOW_TRIGGER: 'sys_flow_trigger',
 
     // Client Development tables
     DL_U_ASSIGNMENT: 'dl_u_assignment',
@@ -156,6 +181,7 @@ const TableNames = {
 
 // String format: one or more path segments separated by '/' (e.g., 'server-development/business-rule' or 'integration/outbound/rest-api')
 const taxonomyItem = z.string().regex(/^[a-z0-9_-]+(\/[a-z0-9_-]+)*$/)
+const flowTaxonomyItem = taxonomyItem.default('automation/flow')
 
 const taxonomyMapping = z
     .object({
@@ -164,12 +190,38 @@ const taxonomyMapping = z
         [TableNames.SYS_HUB_TRIGGER_DEFINITION]: taxonomyItem.default('automation/trigger'),
         [TableNames.SYS_PD_ACTIVITY_DEFINITION]: taxonomyItem.default('automation/activity-definition'),
         [TableNames.SYS_ALIAS]: taxonomyItem.default('automation/workflow-activity'),
-        [TableNames.SYS_HUB_FLOW]: taxonomyItem.default('automation/flow'),
         [TableNames.SYS_DECISION]: taxonomyItem.default('automation/decision-table'),
         [TableNames.SYSEVENT_EMAIL_TEMPLATE]: taxonomyItem.default('automation/email-template'),
         [TableNames.SYSEVENT_EMAIL_ACTION]: taxonomyItem.default('automation/notification'),
         [TableNames.SYS_PD_PROCESS_DEFINITION]: taxonomyItem.default('automation/playbook'),
         [TableNames.SYS_HUB_TRIGGER_TEMPLATE]: taxonomyItem.default('automation/trigger'),
+        [TableNames.SYS_FLOW_EMAIL_TRIGGER]: taxonomyItem.default('automation/trigger'),
+        [TableNames.SYS_FLOW_TRIGGER_PLAN]: taxonomyItem.default('automation/trigger'),
+        [TableNames.SYS_FLOW_KNOWLEDGE_TRIGGER]: taxonomyItem.default('automation/trigger'),
+        [TableNames.SYS_FLOW_RT_QUERY_TRIGGER]: taxonomyItem.default('automation/trigger'),
+        [TableNames.SYS_FLOW_TRIGGER]: taxonomyItem.default('automation/trigger'),
+        [TableNames.SYS_HUB_FLOW]: flowTaxonomyItem,
+        [TableNames.SYS_HUB_ACTION_TYPE_DEFINITION]: flowTaxonomyItem,
+        [TableNames.SYS_ELEMENT_MAPPING]: flowTaxonomyItem,
+        [TableNames.SYS_HUB_FLOW_INPUT]: flowTaxonomyItem,
+        [TableNames.SYS_HUB_FLOW_OUTPUT]: flowTaxonomyItem,
+        [TableNames.SYS_HUB_ACTION_INPUT_ACTION_INSTANCE]: flowTaxonomyItem,
+        [TableNames.SYS_HUB_FLOW_VARIABLE]: flowTaxonomyItem,
+        [TableNames.SYS_HUB_ACTION_INSTANCE]: flowTaxonomyItem,
+        [TableNames.SYS_HUB_SUB_FLOW_INSTANCE]: flowTaxonomyItem,
+        [TableNames.SYS_HUB_FLOW_LOGIC]: flowTaxonomyItem,
+        [TableNames.SYS_HUB_PILL_COMPOUND]: flowTaxonomyItem,
+        [TableNames.SYS_FLOW_CAT_VARIABLE_MODEL]: flowTaxonomyItem,
+        [TableNames.SYS_HUB_TRIGGER_INSTANCE_V2]: flowTaxonomyItem,
+        [TableNames.SYS_HUB_TRIGGER_INSTANCE]: flowTaxonomyItem,
+        [TableNames.SYS_HUB_ACTION_INSTANCE_V2]: flowTaxonomyItem,
+        [TableNames.SYS_HUB_FLOW_LOGIC_INSTANCE_V2]: flowTaxonomyItem,
+        [TableNames.SYS_HUB_SUB_FLOW_INSTANCE_V2]: flowTaxonomyItem,
+        [TableNames.SYS_FLOW_RECORD_TRIGGER]: flowTaxonomyItem,
+        [TableNames.SYS_TRIGGER_RUNNER_MAPPING]: flowTaxonomyItem,
+        [TableNames.SYS_FLOW_TRIGGER_PLAN]: flowTaxonomyItem,
+        [TableNames.SYS_HUB_FLOW_SNAPSHOT]: flowTaxonomyItem,
+        [TableNames.SYS_FLOW_SUBFLOW_PLAN]: flowTaxonomyItem,
 
         // Client Development tables
         [TableNames.DL_U_ASSIGNMENT]: taxonomyItem.default('client-development/assignment-data-lookup'),

package/src/telemetry/clients/abstract-client.ts

@@ -0,0 +1,63 @@
+import type { Telemetry, TelemetryEvent, TelemetryEventData, TimerMetric, InstanceSettings, Session } from '../types'
+import { AppSeeTimerMetric } from './util'
+
+export abstract class AbstractAppSeeClient implements Telemetry {
+    protected readonly sdkVersion?: string | undefined
+    protected readonly clientName: 'cli' | 'ide' | string
+    protected readonly hostname?: string | undefined
+    protected session?: Session
+    protected startPromise?: Promise<void>
+
+    constructor(
+        protected readonly config: InstanceSettings,
+        protected telemetryAttributes: {
+            sdkVersion?: string
+            clientName: 'cli' | 'ide' | string
+            hostname?: string
+        }
+    ) {
+        this.sdkVersion = telemetryAttributes.sdkVersion
+        this.clientName = telemetryAttributes.clientName
+        this.hostname = telemetryAttributes.hostname
+    }
+
+    startTimerEvent(name: TelemetryEvent['name']): TimerMetric {
+        return new AppSeeTimerMetric(name)
+    }
+
+    public start(): Promise<void> {
+        if (!this.startPromise) {
+            this.startPromise = this.doStart()
+        }
+        return this.startPromise
+    }
+
+    public sendEvent(events: TelemetryEvent | TelemetryEvent[]): void {
+        //we fire and forget the telemetry events
+        this.start()
+            .then(() => this.doSendEvent(events))
+            .catch(() => {})
+    }
+
+    public error(event: string, error: unknown, data?: TelemetryEventData) {
+        this.sendEvent([
+            {
+                name: 'error',
+                data: {
+                    event,
+                    ...(error instanceof Error
+                        ? {
+                              name: error.name,
+                              message: error.message,
+                              stack: error.stack,
+                          }
+                        : { message: error?.toString() }),
+                    ...data,
+                },
+            },
+        ])
+    }
+
+    protected abstract doStart(): Promise<void>
+    protected abstract doSendEvent(events: TelemetryEvent | TelemetryEvent[]): void
+}