confluent-schema-registry 3.3.2

package/src/api/middleware/userAgent.spec.ts

@@ -0,0 +1,53 @@
+import { Request } from 'mappersmith'
+
+import UserAgentMiddleware from './userAgent'
+
+const middlewareParams = (clientId?: string) => ({
+  resourceName: 'resourceNameMock',
+  resourceMethod: 'resourceMethodMock',
+  context: { context: 'contextMock' },
+  clientId,
+})
+
+describe('UserAgentMiddleware', () => {
+  let next, request
+
+  beforeEach(() => {
+    request = ({
+      enhance: jest.fn(),
+    } as unknown) as jest.Mocked<Request>
+    next = jest.fn().mockResolvedValue(request)
+  })
+
+  describe('When the user has provided a clientId', () => {
+    const params = middlewareParams('some-client-id')
+
+    it('should add the client id as a user agent comment', async () => {
+      const middleware = UserAgentMiddleware(params)
+
+      await middleware.prepareRequest(next, jest.fn())
+
+      expect(request.enhance).toHaveBeenCalledWith({
+        headers: {
+          'User-Agent': `@kafkajs/confluent-schema-registry (${params.clientId})`,
+        },
+      })
+    })
+  })
+
+  describe('When the user has not provided a clientId', () => {
+    const params = middlewareParams()
+
+    it('should not include a comment in the user agent', async () => {
+      const middleware = UserAgentMiddleware(params)
+
+      await middleware.prepareRequest(next, jest.fn())
+
+      expect(request.enhance).toHaveBeenCalledWith({
+        headers: {
+          'User-Agent': `@kafkajs/confluent-schema-registry`,
+        },
+      })
+    })
+  })
+})

package/src/api/middleware/userAgent.ts

@@ -0,0 +1,19 @@
+import { Middleware } from 'mappersmith'
+import { DEFAULT_API_CLIENT_ID } from '../../constants'
+
+const product = '@kafkajs/confluent-schema-registry'
+
+const userAgentMiddleware: Middleware = ({ clientId }) => {
+  const comment = clientId !== DEFAULT_API_CLIENT_ID ? clientId : undefined
+  const userAgent = comment ? `${product} (${comment})` : product
+  const headers = {
+    'User-Agent': userAgent,
+  }
+  return {
+    prepareRequest: next => {
+      return next().then(req => req.enhance({ headers }))
+    },
+  }
+}
+
+export default userAgentMiddleware

package/src/cache.ts

@@ -0,0 +1,34 @@
+import { AvroSchema, Schema, SchemaType } from './@types'
+
+type CacheEntry = { type: SchemaType; schema: Schema | AvroSchema }
+
+export default class Cache {
+  registryIdBySubject: { [key: string]: number }
+  schemasByRegistryId: { [key: string]: CacheEntry }
+
+  constructor() {
+    this.registryIdBySubject = {}
+    this.schemasByRegistryId = {}
+  }
+
+  getLatestRegistryId = (subject: string): number | undefined => this.registryIdBySubject[subject]
+
+  setLatestRegistryId = (subject: string, id: number): number => {
+    this.registryIdBySubject[subject] = id
+
+    return this.registryIdBySubject[subject]
+  }
+
+  getSchema = (registryId: number): CacheEntry | undefined => this.schemasByRegistryId[registryId]
+
+  setSchema = (registryId: number, type: SchemaType, schema: Schema): CacheEntry => {
+    this.schemasByRegistryId[registryId] = { type, schema }
+
+    return this.schemasByRegistryId[registryId]
+  }
+
+  clear = (): void => {
+    this.registryIdBySubject = {}
+    this.schemasByRegistryId = {}
+  }
+}

package/src/constants.ts

@@ -0,0 +1,13 @@
+export enum COMPATIBILITY {
+  NONE = 'NONE',
+  FULL = 'FULL',
+  BACKWARD = 'BACKWARD',
+  FORWARD = 'FORWARD',
+  BACKWARD_TRANSITIVE = 'BACKWARD_TRANSITIVE',
+  FORWARD_TRANSITIVE = 'FORWARD_TRANSITIVE',
+  FULL_TRANSITIVE = 'FULL_TRANSITIVE',
+}
+
+export const DEFAULT_SEPERATOR = '.'
+
+export const DEFAULT_API_CLIENT_ID = 'Confluent_Schema_Registry'

package/src/errors.ts

@@ -0,0 +1,26 @@
+class ConfluentSchemaRegistryError extends Error {
+  constructor(error: any) {
+    super(error.message || error)
+    this.name = this.constructor.name
+  }
+}
+
+class ConfluentSchemaRegistryArgumentError extends ConfluentSchemaRegistryError {}
+class ConfluentSchemaRegistryCompatibilityError extends ConfluentSchemaRegistryError {}
+class ConfluentSchemaRegistryInvalidSchemaError extends ConfluentSchemaRegistryError {}
+class ConfluentSchemaRegistryValidationError extends ConfluentSchemaRegistryError {
+  public paths: string[][]
+
+  constructor(error: any, paths: string[][]) {
+    super(error)
+    this.paths = paths
+  }
+}
+
+export {
+  ConfluentSchemaRegistryError,
+  ConfluentSchemaRegistryArgumentError,
+  ConfluentSchemaRegistryCompatibilityError,
+  ConfluentSchemaRegistryInvalidSchemaError,
+  ConfluentSchemaRegistryValidationError,
+}

package/src/index.ts

@@ -0,0 +1,4 @@
+export { default as SchemaRegistry } from './SchemaRegistry'
+export * from './utils'
+export { SchemaType } from './@types'
+export { COMPATIBILITY } from './constants'

package/src/schemaTypeResolver.ts

@@ -0,0 +1,101 @@
+import AvroHelper from './AvroHelper'
+import JsonHelper from './JsonHelper'
+import JsonSchema from './JsonSchema'
+import ProtoHelper from './ProtoHelper'
+import ProtoSchema from './ProtoSchema'
+import {
+  SchemaType,
+  SchemaHelper,
+  ConfluentSchema,
+  SchemaRegistryAPIClientOptions,
+  LegacyOptions,
+  ProtocolOptions,
+  AvroOptions,
+  JsonOptions,
+  ProtoOptions,
+  Schema,
+  AvroSchema,
+} from './@types'
+import { ConfluentSchemaRegistryArgumentError } from './errors'
+
+const helperTypeFromSchemaTypeMap: Record<string, SchemaHelper> = {}
+
+export const schemaTypeFromString = (schemaTypeString: string) => {
+  switch (schemaTypeString) {
+    case 'AVRO':
+    case undefined:
+      return SchemaType.AVRO
+    case 'JSON':
+      return SchemaType.JSON
+    case 'PROTOBUF':
+      return SchemaType.PROTOBUF
+    default:
+      return SchemaType.UNKNOWN
+  }
+}
+
+export const helperTypeFromSchemaType = (
+  schemaType: SchemaType = SchemaType.AVRO,
+): SchemaHelper => {
+  const schemaTypeStr = schemaType.toString()
+
+  if (!helperTypeFromSchemaTypeMap[schemaTypeStr]) {
+    let helper
+    switch (schemaType) {
+      case SchemaType.AVRO: {
+        helper = new AvroHelper()
+        break
+      }
+      case SchemaType.JSON: {
+        helper = new JsonHelper()
+        break
+      }
+      case SchemaType.PROTOBUF: {
+        helper = new ProtoHelper()
+        break
+      }
+      default:
+        throw new ConfluentSchemaRegistryArgumentError('invalid schemaType')
+    }
+    helperTypeFromSchemaTypeMap[schemaTypeStr] = helper
+  }
+  return helperTypeFromSchemaTypeMap[schemaTypeStr]
+}
+
+export const schemaFromConfluentSchema = (
+  confluentSchema: ConfluentSchema,
+  options?: SchemaRegistryAPIClientOptions,
+): Schema | AvroSchema => {
+  try {
+    let schema: Schema
+
+    switch (confluentSchema.type) {
+      case SchemaType.AVRO: {
+        const opts: AvroOptions | undefined =
+          (options as LegacyOptions)?.forSchemaOptions ||
+          (options as ProtocolOptions)?.[SchemaType.AVRO]
+        schema = (helperTypeFromSchemaType(confluentSchema.type) as AvroHelper).getAvroSchema(
+          confluentSchema,
+          opts,
+        )
+        break
+      }
+      case SchemaType.JSON: {
+        const opts: JsonOptions | undefined = (options as ProtocolOptions)?.[SchemaType.JSON]
+        schema = new JsonSchema(confluentSchema, opts)
+        break
+      }
+      case SchemaType.PROTOBUF: {
+        const opts: ProtoOptions | undefined = (options as ProtocolOptions)?.[SchemaType.PROTOBUF]
+        schema = new ProtoSchema(confluentSchema, opts)
+        break
+      }
+      default:
+        throw new ConfluentSchemaRegistryArgumentError('invalid schemaType')
+    }
+
+    return schema
+  } catch (err) {
+    throw new ConfluentSchemaRegistryArgumentError(err.message)
+  }
+}

package/src/utils/avdlToAVSC.spec.ts

@@ -0,0 +1,79 @@
+import path from 'path'
+import fs from 'fs-extra'
+import execa from 'execa'
+import avro from 'avsc'
+
+import SchemaRegistry from '../SchemaRegistry'
+import { avdlToAVSCAsync } from './avdlToAVSC'
+
+const registry = new SchemaRegistry({ host: 'http://localhost:8982' })
+const absolutePath = (...paths: string[]) => path.join(__dirname, '../..', ...paths)
+
+const compareWithJavaImplementation = (avdlPath: string, name: string) => async () => {
+  const absolutePathToAvdlToAVSC = absolutePath('./bin/avdlToAVSC.sh')
+  const execaArgs = [`./fixtures/avdl/${avdlPath}`, name]
+
+  let expectedAVSC
+  try {
+    const { stdout: result } = await execa(absolutePathToAvdlToAVSC, execaArgs)
+    expectedAVSC = JSON.parse(result)
+  } catch (error) {
+    console.error(`Error when running ${absolutePathToAvdlToAVSC}`, error) // eslint-disable-line no-console
+    throw error
+  }
+
+  const avsc = await avdlToAVSCAsync(absolutePath('./fixtures/avdl', avdlPath))
+
+  expect(avsc).toEqual(expectedAVSC)
+  expect(avro.Type.forSchema(avsc)).toBeTruthy()
+  expect(await registry.register(avsc)).toBeTruthy()
+}
+
+beforeAll(async () => {
+  jest.setTimeout(10000)
+  await fs.emptyDir(absolutePath('./tmp'))
+})
+
+test('simple protocol', compareWithJavaImplementation('simple.avdl', 'Simple'))
+
+test('protocol with two levels', compareWithJavaImplementation('two.avdl', 'Two'))
+
+test('protocol with multiple levels', compareWithJavaImplementation('multiple.avdl', 'Multiple'))
+
+test('protocol with union', compareWithJavaImplementation('union.avdl', 'Union'))
+
+test(
+  'protocol with multiple union levels',
+  compareWithJavaImplementation('multiple_union.avdl', 'MultipleUnion'),
+)
+
+test('protocol with enum', compareWithJavaImplementation('enum.avdl', 'Enum'))
+
+test('protocol with enum & union', compareWithJavaImplementation('enum_union.avdl', 'EnumUnion'))
+
+test('protocol with array', compareWithJavaImplementation('array.avdl', 'Array'))
+
+test('protocol with array & union', compareWithJavaImplementation('array_union.avdl', 'ArrayUnion'))
+
+test('protocol with really complex stuff', compareWithJavaImplementation('complex.avdl', 'Complex'))
+
+test(
+  'protocol with multiple namespaces',
+  compareWithJavaImplementation('multiple_namespaces.avdl', 'MultipleNamespaces'),
+)
+
+/*
+ * AVSC includes the namespace of imported records even if they are being imported
+ * into the same namespace, causing a difference against the Java version.
+ *
+ * @issue: https://github.com/mtth/avsc/issues/281
+ */
+test.skip(
+  'protocol with import from same namespace',
+  compareWithJavaImplementation('import.avdl', 'Import'),
+)
+
+test(
+  'protocol with import from different namespace',
+  compareWithJavaImplementation('import_multiple_namespaces.avdl', 'ImportMultipleNamespaces'),
+)

package/src/utils/avdlToAVSC.ts

@@ -0,0 +1,106 @@
+import * as fs from 'fs'
+import { assembleProtocol, readProtocol } from 'avsc'
+
+import { ConfluentSchemaRegistryError } from '../errors'
+
+interface AssembleProtocolError extends Error {
+  path: string
+}
+interface Obj {
+  [key: string]: any
+}
+interface Iterable extends Obj {
+  map: any
+}
+interface Field {
+  type: {
+    type: string
+    items: any
+  }
+}
+
+let cache: any
+const merge = Object.assign
+const isObject = (obj: unknown): obj is Obj => obj && typeof obj === 'object'
+const isIterable = (obj: unknown): obj is Iterable =>
+  isObject(obj) && typeof obj.map !== 'undefined'
+const isFieldArray = (field: unknown): field is Field =>
+  isObject(field) && isObject(field.type) && field.type.type === 'array'
+
+const combine = (rootType: any, types: any) => {
+  if (!rootType.fields) {
+    return rootType
+  }
+
+  const find = (name: any) => {
+    if (typeof name === 'string') {
+      name = name.toLowerCase()
+    }
+
+    const typeToCombine = types.find((t: any) => {
+      const names = []
+      if (t.namespace) {
+        names.push(`${t.namespace}.`)
+      }
+      names.push(t.name.toLowerCase())
+
+      return names.join('') === name
+    })
+
+    if (!typeToCombine || cache[typeToCombine.name]) {
+      return null
+    }
+
+    cache[typeToCombine.name] = 1
+
+    return combine(typeToCombine, types)
+  }
+
+  const combinedFields = rootType.fields.map((field: any) => {
+    if (isFieldArray(field)) {
+      const typeToCombine = find(field.type.items)
+      return typeToCombine
+        ? merge(field, { type: merge(field.type, { items: typeToCombine }) })
+        : field
+    } else if (isIterable(field.type)) {
+      const type = field.type.map((unionType: any) => {
+        if (isObject(unionType)) {
+          const typeToCombine = find(unionType.items)
+          return typeToCombine ? merge(unionType, { items: typeToCombine }) : unionType
+        } else {
+          return find(unionType) || unionType
+        }
+      })
+
+      return merge(field, { type })
+    }
+
+    const typeToCombine = find(field.type)
+    return typeToCombine ? merge(field, { type: typeToCombine }) : field
+  })
+
+  return merge(rootType, { fields: combinedFields })
+}
+
+export function avdlToAVSC(path: any) {
+  cache = {}
+  const protocol = readProtocol(fs.readFileSync(path, 'utf8'))
+
+  return merge({ namespace: protocol.namespace }, combine(protocol.types.pop(), protocol.types))
+}
+
+export async function avdlToAVSCAsync(path: string) {
+  cache = {}
+
+  const protocol: { [key: string]: any } = await new Promise((resolve, reject) => {
+    assembleProtocol(path, (err: AssembleProtocolError, schema) => {
+      if (err) {
+        reject(new ConfluentSchemaRegistryError(`${err.message}. Caused by: ${err.path}`))
+      } else {
+        resolve(schema)
+      }
+    })
+  })
+
+  return merge({ namespace: protocol.namespace }, combine(protocol.types.pop(), protocol.types))
+}

package/src/utils/index.ts

@@ -0,0 +1,2 @@
+export { avdlToAVSC, avdlToAVSCAsync } from './avdlToAVSC'
+export { readAVSC, readAVSCAsync } from './readAVSC'

package/src/utils/readAVSC.spec.ts

@@ -0,0 +1,23 @@
+import path from 'path'
+
+import { readAVSC, readAVSCAsync } from './readAVSC'
+import { ConfluentSchemaRegistryInvalidSchemaError } from '../errors'
+
+describe('readAVSC', () => {
+  const invalidSchemaFiles = ['invalidType', 'missingFields', 'missingName', 'missingType']
+  invalidSchemaFiles.forEach(schemaName => {
+    it(`throws an exception for invalid schema definitions - ${schemaName}`, () => {
+      expect(() =>
+        readAVSC(path.join(__dirname, `../../fixtures/avsc/invalid/${schemaName}.avsc`)),
+      ).toThrow(ConfluentSchemaRegistryInvalidSchemaError)
+    })
+  })
+})
+
+describe('readAVSCAsync', () => {
+  it('returns a validated schema asynchronously', async () => {
+    return expect(
+      readAVSCAsync(path.join(__dirname, `../../fixtures/avsc/person.avsc`)),
+    ).resolves.toHaveProperty('name', 'Person')
+  })
+})

package/src/utils/readAVSC.ts

@@ -0,0 +1,36 @@
+import fs from 'fs'
+import { promisify } from 'util'
+
+import { RawAvroSchema } from '../@types'
+import { ConfluentSchemaRegistryInvalidSchemaError } from '../errors'
+
+const readFileAsync = promisify(fs.readFile)
+const ENCODING = 'utf-8'
+
+function isValidSchema(rawSchema: any): rawSchema is RawAvroSchema {
+  return (
+    'name' in rawSchema &&
+    'type' in rawSchema &&
+    rawSchema.type === 'record' &&
+    'fields' in rawSchema
+  )
+}
+
+function validatedSchema(path: string, rawSchema: any): RawAvroSchema {
+  if (!isValidSchema(rawSchema)) {
+    throw new ConfluentSchemaRegistryInvalidSchemaError(
+      `${path} is not recognized as a valid AVSC file (expecting valid top-level name, type and fields attributes)`,
+    )
+  }
+  return rawSchema
+}
+
+export function readAVSC(path: string): RawAvroSchema {
+  const rawSchema = JSON.parse(fs.readFileSync(path, ENCODING))
+  return validatedSchema(path, rawSchema)
+}
+
+export async function readAVSCAsync(path: string): Promise<RawAvroSchema> {
+  const rawSchema = JSON.parse(await readFileAsync(path, ENCODING))
+  return validatedSchema(path, rawSchema)
+}

package/src/wireDecoder.ts

@@ -0,0 +1,5 @@
+export default (buffer: Buffer) => ({
+  magicByte: buffer.slice(0, 1),
+  registryId: buffer.slice(1, 5).readInt32BE(0),
+  payload: buffer.slice(5, buffer.length),
+})

package/src/wireEncoder.ts

@@ -0,0 +1,10 @@
+const DEFAULT_OFFSET = 0
+
+export const MAGIC_BYTE = Buffer.alloc(1)
+
+export const encode = (registryId: number, payload: Buffer) => {
+  const registryIdBuffer = Buffer.alloc(4)
+  registryIdBuffer.writeInt32BE(registryId, DEFAULT_OFFSET)
+
+  return Buffer.concat([MAGIC_BYTE, registryIdBuffer, payload])
+}

package/tsconfig.json

@@ -0,0 +1,22 @@
+{
+  "compilerOptions": {
+    "baseUrl": ".",
+    "paths": {
+      "*": ["src/*", "declarations/*"]
+    },
+    "outDir": "dist",
+    "lib": ["es2019", "dom"],
+    "module": "commonjs",
+    "moduleResolution": "node",
+    "sourceMap": true,
+    "strict": true,
+    "target": "es2019",
+    "noUnusedParameters": true,
+    "noUnusedLocals": true,
+    "esModuleInterop": true,
+    "declaration": true,
+    "resolveJsonModule": true
+  },
+  "include": ["./src/**/*.ts", "declarations/**/*.ts"],
+  "exclude": ["dist", "node_modules", "**/*.spec.ts"]
+}