@servicenow/sdk-build-plugins 4.1.1 → 4.3.0

package/src/list-plugin.ts

@@ -6,6 +6,7 @@ import {
     StringShape,
     type ObjectShape,
     type Diagnostics,
+    Database,
 } from '@servicenow/sdk-build-core'
 import { create } from 'xmlbuilder2'
 import { generateDeprecatedDiagnostics } from './utils'
@@ -33,6 +34,35 @@ function generateListColumnProps(column: ObjectShape, index: number, diagnostics
     return props
 }
 
+function getListUpdateName(list: Record): string {
+    const { view: viewShape, parent: parentShape, relationship: relShape, name: tableShape } = list.properties()
+    const viewRecordId = viewShape?.isRecord() ? viewShape.getId() : viewShape?.isRecordId() ? viewShape : undefined
+    const parent = parentShape?.isString() ? parentShape.asString().getValue() : undefined
+    const relationship = relShape?.ifDefined()
+        ? (relShape.ifString()?.getValue() ?? relShape.asRecord().getId().getValue())
+        : undefined
+    const table = tableShape?.isString() ? tableShape.asString().getValue() : undefined
+    let view = viewRecordId?.isDefined()
+        ? ((viewRecordId?.getKey('name') as string) ?? 'NULL')
+        : viewShape?.asString().getValue()
+    view =
+        viewRecordId && (viewRecordId.getKey('name') === DEFAULT_VIEW || viewRecordId.getValue() === DEFAULT_VIEW)
+            ? 'NULL'
+            : view
+
+    const updateNameFormats = new Map([
+        ['no-parent-no-relationship', [table, view]],
+        ['parent-no-relationship', [table, parent, view]],
+        ['no-parent-relationship', ['NULL', relationship, view]],
+        ['parent-relationship', [parent, relationship, view]],
+    ])
+
+    const key = `${parent ? 'parent' : 'no-parent'}-${relationship ? 'relationship' : 'no-relationship'}`
+    const segments = updateNameFormats.get(key) ?? []
+
+    return ['sys_ui_list', ...segments].join('_').toLocaleLowerCase()
+}
+
 export const ListPlugin = Plugin.create({
     name: 'ListPlugin',
     records: {
@@ -49,6 +79,7 @@ export const ListPlugin = Plugin.create({
                     inverse: true,
                 },
             },
+            getUpdateName: (list) => ({ success: true, value: getListUpdateName(list) }),
             toShape(record, { descendants }) {
                 return {
                     success: true,
@@ -160,12 +191,71 @@ export const ListPlugin = Plugin.create({
                     success: true,
                     value: {
                         source: list,
-                        name: `sys_ui_list_${list.getId().getValue()}.xml`,
+                        name: `${getListUpdateName(list)}.xml`,
                         category: list.getInstallCategory(),
                         content: xml.end({ prettyPrint: true }),
                     },
                 }
             },
+            async diff(existing, incoming, _, { factory }) {
+                if (incoming.query().length === 0 || existing.query().length === 0) {
+                    return {
+                        success: true,
+                        value: incoming.query().length === 0 ? new Database() : new Database(incoming.query()),
+                    }
+                }
+                const changeDatabase = new Database()
+                let hasChanges = false
+                const existingList = existing.query('sys_ui_list')[0]
+                const incomingList = incoming.query('sys_ui_list')[0]
+                const existingElements = existing.query('sys_ui_list_element')
+                const incomingElements = incoming.query('sys_ui_list_element')
+
+                if (incomingList && existingList) {
+                    if (!existingList.strictEquals(incomingList)) {
+                        changeDatabase.insert(existingList.merge(incomingList))
+                        hasChanges = true
+                    }
+                } else if (incomingList) {
+                    changeDatabase.insert(incomingList)
+                    hasChanges = true
+                }
+
+                const markForRemoval: Record[] = []
+                for (const element of existingElements) {
+                    const match = incoming.resolve(element.getId())
+                    if (!match) {
+                        hasChanges = true
+                        markForRemoval.push(element)
+                    } else {
+                        hasChanges = hasChanges || !element.strictEquals(match)
+                        changeDatabase.insert(element.merge(match))
+                    }
+                }
+
+                incomingElements.forEach((element) => {
+                    const match = changeDatabase.resolve(element.getId())
+                    if (!match) {
+                        changeDatabase.insert(element)
+                    }
+                })
+
+                for (const element of markForRemoval) {
+                    const deleteRecord = await factory.createRecord({
+                        source: element.getSource(),
+                        table: 'sys_ui_list_element',
+                        explicitId: element.getId(),
+                        properties: element.properties(),
+                        action: 'DELETE',
+                    })
+                    changeDatabase.insert(deleteRecord)
+                }
+
+                return {
+                    success: true,
+                    value: hasChanges ? changeDatabase : new Database(),
+                }
+            },
         },
         sys_ui_list_element: {
             coalesce: ['list_id', 'element'],

package/src/now-attach-plugin.ts

@@ -0,0 +1,403 @@
+import {
+    path as pathModule,
+    CallExpressionShape,
+    FileSystem,
+    Plugin,
+    type Source,
+    type Record,
+    crypto,
+    path,
+    type Factory,
+    type AssociateRecordsShape,
+    type LazyValue,
+    gunzip,
+    ObjectShape,
+    ts,
+    gzipSync,
+    isLazyValue,
+    hasAssociatedRecords,
+    unloadBuilder,
+} from '@servicenow/sdk-build-core'
+import { CHUNK_SIZE, chunkData, generateId } from './static-content-plugin'
+
+export async function sha256(message: Buffer) {
+    const hashBuffer = await crypto.subtle.digest('SHA-256', new Uint8Array(message))
+    const hashArray = Array.from(new Uint8Array(hashBuffer))
+    return hashArray.map((b) => b.toString(16).padStart(2, '0')).join('')
+}
+
+const ImageAttachmentTypes = new Map([
+    ['.jpg', 'image/jpeg'],
+    ['.jpeg', 'image/jpeg'],
+    ['.png', 'image/png'],
+    ['.bmp', 'image/bmp'],
+    ['.gif', 'image/gif'],
+    ['.ico', 'image/ico'],
+    ['.svg', 'image/svg+xml'],
+])
+
+function getExtension(contentType: string) {
+    switch (contentType) {
+        case 'image/jpeg':
+            return '.jpg'
+        case 'image/png':
+            return '.png'
+        case 'image/bmp':
+            return '.bmp'
+        case 'image/gif':
+            return '.gif'
+        case 'image/vnd.microsoft.icon':
+        case 'image/x-icon':
+        case 'image/ico':
+            return '.ico'
+        case 'image/svg':
+        case 'image/svg+xml':
+            return '.svg'
+        default:
+            return ''
+    }
+}
+
+type BaseAttachmentProperties = {
+    average_image_color: ''
+    chunk_size_bytes: unknown
+    compressed: boolean
+    content_type: string | undefined
+    hash: string
+    image_height: ''
+    image_width: ''
+    size_bytes: unknown
+    size_compressed: unknown
+    data: string[]
+}
+
+export class NowAttachShape extends CallExpressionShape implements AssociateRecordsShape, LazyValue {
+    private readonly baseProperties: BaseAttachmentProperties
+
+    constructor({
+        source,
+        path,
+        baseProperties,
+    }: { source: Source; path: string; baseProperties: BaseAttachmentProperties }) {
+        super({ source, callee: 'Now.attach', args: [path] })
+
+        this.baseProperties = baseProperties
+    }
+
+    evaluate(parentRecord: Record, field: string) {
+        return generateId(parentRecord.getId().getValue(), 'sys_attachment', this.baseProperties.hash, field)
+    }
+
+    getPath(): string {
+        return this.getArgument(0).asString().getValue()
+    }
+
+    getBaseAttachmentProperties() {
+        return this.baseProperties
+    }
+
+    async getBufferData() {
+        const buffers = await Promise.all(this.getBaseAttachmentProperties().data.map((d) => Buffer.from(d, 'base64')))
+        const combinedBuffer = Buffer.concat(buffers)
+        return await gunzip(combinedBuffer)
+    }
+
+    async createAssociatedRecords({
+        parentRecord,
+        factory,
+        field,
+    }: {
+        parentRecord: Record
+        factory: Factory
+        field: string
+    }) {
+        const attachment = await factory.createRecord({
+            source: parentRecord.getSource(),
+            table: 'sys_attachment',
+            properties: {
+                sys_id: generateId(parentRecord.getId().getValue(), 'sys_attachment', this.baseProperties.hash, field),
+                average_image_color: this.baseProperties.average_image_color,
+                chunk_size_bytes: this.baseProperties.chunk_size_bytes,
+                compressed: this.baseProperties.compressed,
+                content_type: this.baseProperties.content_type,
+                hash: this.baseProperties.hash,
+                image_height: this.baseProperties.image_height,
+                image_width: this.baseProperties.image_width,
+                size_bytes: this.baseProperties.size_bytes,
+                size_compressed: this.baseProperties.size_compressed,
+                file_name: field,
+                table_name: `ZZ_YY${parentRecord.getTable()}`,
+                table_sys_id: parentRecord.getId().getValue(),
+            },
+        })
+
+        const attachmentDocs = await Promise.all(
+            this.baseProperties.data.map((data, i) =>
+                factory.createRecord({
+                    source: parentRecord.getSource(),
+                    table: 'sys_attachment_doc',
+                    properties: {
+                        sys_id: generateId(parentRecord.getId().getValue(), 'sys_attachment_doc', field, data),
+                        data,
+                        length: data.length,
+                        position: i,
+                        sys_attachment: attachment.getId().getValue(),
+                    },
+                })
+            )
+        )
+
+        return [attachment, ...attachmentDocs]
+    }
+
+    override equals<E = unknown>(value: E): boolean {
+        return value instanceof NowAttachShape && value.baseProperties.hash === this.baseProperties.hash
+    }
+
+    static async create(source: Source, fs: FileSystem, filePath: string) {
+        const buffer = (await fs.readFileSync(filePath)) as Buffer
+
+        const hash = await sha256(buffer)
+        const gzipped = gzipSync(buffer)
+        const compressedFileData = Buffer.from(gzipped)
+
+        // The first sys_attachment_doc (position=0), contains only the
+        // first 10 bytes of the gzip header, which are base64-encoded into 16 characters.
+        // For more info on the gzip header: https://www.ietf.org/rfc/rfc1952.txt
+        const gzipHeader = compressedFileData.subarray(0, 10).toString('base64')
+        const remainingCompressedData = chunkData(compressedFileData.subarray(10).toString('base64'))
+
+        return new NowAttachShape({
+            source,
+            path: filePath,
+            baseProperties: {
+                average_image_color: '',
+                chunk_size_bytes: CHUNK_SIZE,
+                compressed: true,
+                content_type: ImageAttachmentTypes.get(path.extname(filePath).toLowerCase()),
+                hash,
+                image_height: '',
+                image_width: '',
+                size_bytes: buffer.length,
+                size_compressed: gzipped.length,
+                data: [gzipHeader, ...remainingCompressedData],
+            },
+        })
+    }
+
+    static fromAttachmentRecord(parentUpdateName: string, attachment: Record, attachmentDocs: Record[]) {
+        const field = attachment.get('file_name').getValue()
+        const extension = getExtension(attachment.get('content_type').toString().getValue() || '')
+        const relativePath = `.${path.sep}${parentUpdateName}_${field}${extension}`
+
+        return new NowAttachShape({
+            source: attachment,
+            path: relativePath,
+            baseProperties: {
+                average_image_color: '',
+                chunk_size_bytes: attachment.get('chunk_size_bytes').getValue(),
+                compressed: attachment.get('compressed').toBoolean().getValue(),
+                content_type: attachment.get('content_type').toString().getValue() || undefined,
+                hash: attachment.get('hash').toString().getValue(),
+                image_height: '',
+                image_width: '',
+                size_bytes: attachment.get('size_bytes').getValue(),
+                size_compressed: attachment.get('size_compressed').getValue(),
+                data: attachmentDocs.map((doc) => doc.get('data').toString().getValue()),
+            },
+        })
+    }
+}
+
+export const NowAttachPlugin = Plugin.create({
+    name: 'NowAttachPlugin',
+    records: {
+        '*': {
+            async toFile(record, { config, database, factory, transform }) {
+                const recordBuilder = unloadBuilder(config)
+                const updateName = await transform.getUpdateName(record)
+                const builder = recordBuilder.record(record, updateName)
+                let attachmentsProcessed = false
+
+                record
+                    .entries()
+                    .sort(([a], [b]) => a.localeCompare(b)) // Sort keys to make outputs more deterministic
+                    .forEach(([prop, shape]) => {
+                        if (isLazyValue(shape)) {
+                            attachmentsProcessed = true
+                            builder.field(prop, shape.evaluate(record, prop))
+                        } else {
+                            builder.field(prop, shape)
+                        }
+                    })
+
+                for (const field in record.properties()) {
+                    const shape = record.get(field)
+                    if (hasAssociatedRecords(shape)) {
+                        attachmentsProcessed = true
+                        for (const rec of await shape.createAssociatedRecords({
+                            parentRecord: record,
+                            factory,
+                            field,
+                        })) {
+                            const claimUpdateName = await transform.getUpdateName(rec)
+                            const claimBuilder = recordBuilder.record(rec, claimUpdateName)
+                            rec.entries()
+                                .sort(([a], [b]) => a.localeCompare(b)) // Sort keys to make outputs more deterministic
+                                .forEach(([prop, shape]) => claimBuilder.field(prop, shape))
+                        }
+                    }
+                }
+
+                if (!attachmentsProcessed) {
+                    return {
+                        success: false,
+                    }
+                }
+
+                const claims = database
+                    .query('sys_claim')
+                    .filter((claim) => claim.get('metadata_update_name').equals(updateName))
+
+                for (const claim of claims) {
+                    const claimUpdateName = await transform.getUpdateName(claim)
+                    const claimBuilder = recordBuilder.record(claim, claimUpdateName)
+                    claim
+                        .entries()
+                        .sort(([a], [b]) => a.localeCompare(b)) // Sort keys to make outputs more deterministic
+                        .forEach(([prop, shape]) => claimBuilder.field(prop, shape))
+                }
+
+                return {
+                    success: true,
+                    value: {
+                        source: record,
+                        name: `${updateName}.xml`,
+                        category: record.getInstallCategory(),
+                        content: recordBuilder.end(),
+                    },
+                }
+            },
+        },
+        sys_attachment: {
+            toShape: async (record) => {
+                const fileName = record.get('file_name').getValue()
+                const hash = record.get('hash').getValue()
+                const extension = getExtension(record.get('content_type').toString().getValue() || '')
+
+                return {
+                    success: true,
+                    value: new CallExpressionShape({
+                        source: record,
+                        args: [`${fileName}_${hash}${extension}`],
+                        callee: 'Now.attach',
+                    }),
+                }
+            },
+        },
+    },
+    shapes: [
+        {
+            shape: CallExpressionShape,
+            fileTypes: ['fluent'],
+            async toSubclass(callExpression, { diagnostics, project, fs }) {
+                if (callExpression.getCallee() !== 'Now.attach') {
+                    return { success: false }
+                }
+
+                const arg = callExpression.getArgument(0)
+                if (!arg.isString()) {
+                    diagnostics.error(arg, 'Now.attach() must have a string argument')
+                    return { success: false }
+                }
+
+                const path = arg.getValue()
+                if (!/^\.\.?\/.*$/.test(path.trim()) && path.includes('/')) {
+                    diagnostics.error(arg, 'Now.attach() argument must be a relative path')
+                    return { success: false }
+                }
+
+                const absolutePath = pathModule.resolve(pathModule.dirname(callExpression.getOriginalFilePath()), path)
+
+                if (pathModule.relative(project.getRootDir(), absolutePath).startsWith('..')) {
+                    diagnostics.error(arg, `File path is not within project: ${absolutePath}`)
+                    return { success: false }
+                }
+
+                if (!FileSystem.existsSync(fs, absolutePath)) {
+                    diagnostics.error(arg, 'File not found')
+                    return { success: false }
+                }
+
+                const attachment = await NowAttachShape.create(callExpression.getSource(), fs, absolutePath)
+
+                return {
+                    success: true,
+                    value: attachment,
+                }
+            },
+        },
+        {
+            shape: NowAttachShape,
+            async commit(shape, target, { transform, fs }) {
+                const targetResult = await transform.toShape(target)
+                if (!targetResult.success) {
+                    return { success: false }
+                }
+
+                const { value: targetShape } = targetResult
+                let absolutePath: string
+
+                if (
+                    targetShape.is(NowAttachShape) &&
+                    path.extname(targetShape.getPath()) === path.extname(shape.getPath())
+                ) {
+                    absolutePath = targetShape.getPath()
+                } else {
+                    absolutePath = path.resolve(path.dirname(target.getSourceFile().getFilePath()), shape.getPath())
+                    target.replaceWithText(shape.getCode())
+                }
+
+                fs.mkdirSync(path.dirname(absolutePath), { recursive: true })
+                const bufferData = await shape.getBufferData()
+                if (!FileSystem.existsSync(fs, absolutePath) || !fs.readFileSync(absolutePath).equals(bufferData)) {
+                    fs.writeFileSync(absolutePath, bufferData, { encoding: 'binary' })
+                }
+
+                return { success: true }
+            },
+        },
+        // The following is janky. The problem is, in the case of a transform where
+        // the ObjectLiteralExpression for a plugin CallExpression exists, but doesn't
+        // have the field yet (and will because it's coming from an incoming XML file),
+        // the commit code which saves the actual file (like say a .jpg file) is never
+        // called. Rather, the field is populated via getCode() instead which doesn't
+        // actually save the attached file. This commit is called before the basic-syntax-plugin's
+        // ObjectShape commiter, so we populate the Now.attach field so that the NowAttachPlugin's
+        // commit functiong gets called.
+        {
+            shape: ObjectShape,
+            async commit(shape, target) {
+                if (!ts.Node.isObjectLiteralExpression(target)) {
+                    return { success: false }
+                }
+                const attachShapes = shape
+                    .entries({ resolve: false })
+                    .filter(([name, value]) => value.is(NowAttachShape) && !target.getProperty(name))
+                    .map(([name, shape]) => {
+                        return {
+                            name: ObjectShape.quotePropertyNameIfNeeded(name),
+                            initializer: shape.getCode(),
+                            kind: ts.StructureKind.PropertyAssignment,
+                        }
+                    }) as ts.PropertyAssignmentStructure[]
+
+                if (attachShapes.length > 0) {
+                    target.addPropertyAssignments(attachShapes)
+                }
+
+                return { success: false }
+            },
+        },
+    ],
+})

package/src/now-config-plugin.ts

@@ -130,6 +130,10 @@ export const NowConfigPlugin = Plugin.create({
                     return { success: false }
                 }
 
+                if (rawConfig.type === 'configuration') {
+                    return { success: false }
+                }
+
                 const config = Shape.from(file.getJson(), rawConfig).asObject()
                 const scope = config.get('scope').asString().getValue()
 
@@ -139,8 +143,8 @@ export const NowConfigPlugin = Plugin.create({
                 let packageJsonPath: string
                 try {
                     packageJsonPath = NowConfig.moduleResolutionPath(rawConfig, packageJson, false, 'package.json')
-                } catch (e: any) {
-                    diagnostics.error(file, e.message)
+                } catch (e) {
+                    diagnostics.error(file, (e as Error).message)
                 }
 
                 const accessControls = config.get('accessControls').ifDefined()?.asObject()

package/src/now-include-plugin.ts

@@ -9,6 +9,7 @@ import {
     Shape,
     FileSystem,
     type Transform,
+    taxonomySchemaDefault,
 } from '@servicenow/sdk-build-core'
 import { CallExpressionPlugin } from './call-expression-plugin'
 
@@ -43,9 +44,15 @@ export class NowIncludeShape extends CallExpressionShape {
     }
 
     static async fromRecord(record: Record, text: string | Shape, transform: Transform): Promise<NowIncludeShape> {
+        const scriptType = taxonomySchemaDefault[record.getTable()]
+        const scriptIdentifier = scriptType?.startsWith('client-development')
+            ? '.client'
+            : scriptType?.startsWith('server-development')
+              ? '.server'
+              : ''
         return new NowIncludeShape({
             source: record,
-            path: `./${await transform.getUpdateName(record)}.js`,
+            path: `./${await transform.getUpdateName(record)}${scriptIdentifier}.js`,
             includedText: text instanceof Shape ? text.toString().getValue() : text,
         })
     }

package/src/package-json-plugin.ts

@@ -16,7 +16,7 @@ export const PackageJsonPlugin = Plugin.create({
         {
             shape: JsonFileShape,
             async toRecord(file, { config, packageJson, project, factory, fs, logger }) {
-                if (file.getBaseName() !== 'package.json') {
+                if (file.getBaseName() !== 'package.json' || config.type === 'configuration') {
                     return { success: false }
                 }
 
@@ -45,7 +45,7 @@ export const PackageJsonPlugin = Plugin.create({
                                 path: sysModulePath,
                                 content: Shape.from(
                                     file,
-                                    await fixDependencyVersion(file.getJson(), {
+                                    await fixDependencyVersion(file.getJson().asObject(), {
                                         fs,
                                         logger,
                                         rootDir: project.getRootDir(),
@@ -71,7 +71,7 @@ async function fixDependencyVersion(json: ObjectShape, context: { fs: FileSystem
     }
 
     const repack = await RepackService.create(context.logger, context.fs, context.rootDir)
-    const resolvedDeps = {}
+    const resolvedDeps: { [key: string]: string } = {}
     for (const dep of deps.keys()) {
         const { name, version } = await repack.getResolvedDepDetails(dep)
         resolvedDeps[name] = version