confluent-schema-registry 3.3.2

package/src/SchemaRegistry.ts

@@ -0,0 +1,387 @@
+import { Type } from 'avsc'
+import { Response } from 'mappersmith'
+
+import { encode, MAGIC_BYTE } from './wireEncoder'
+import decode from './wireDecoder'
+import { COMPATIBILITY, DEFAULT_SEPERATOR } from './constants'
+import API, { SchemaRegistryAPIClientArgs, SchemaRegistryAPIClient } from './api'
+import Cache from './cache'
+import {
+  ConfluentSchemaRegistryError,
+  ConfluentSchemaRegistryArgumentError,
+  ConfluentSchemaRegistryCompatibilityError,
+  ConfluentSchemaRegistryValidationError,
+} from './errors'
+import {
+  Schema,
+  RawAvroSchema,
+  AvroSchema,
+  SchemaType,
+  ConfluentSchema,
+  ConfluentSubject,
+  SchemaRegistryAPIClientOptions,
+  AvroConfluentSchema,
+  SchemaResponse,
+  ProtocolOptions,
+  SchemaHelper,
+  SchemaReference,
+  LegacyOptions,
+} from './@types'
+import {
+  helperTypeFromSchemaType,
+  schemaTypeFromString,
+  schemaFromConfluentSchema,
+} from './schemaTypeResolver'
+
+export interface RegisteredSchema {
+  id: number
+}
+
+interface Opts {
+  compatibility?: COMPATIBILITY
+  separator?: string
+  subject: string
+}
+
+interface AvroDecodeOptions {
+  readerSchema?: RawAvroSchema | AvroSchema | Schema
+}
+interface DecodeOptions {
+  [SchemaType.AVRO]?: AvroDecodeOptions
+}
+
+const DEFAULT_OPTS = {
+  compatibility: COMPATIBILITY.BACKWARD,
+  separator: DEFAULT_SEPERATOR,
+}
+export default class SchemaRegistry {
+  private api: SchemaRegistryAPIClient
+  private cacheMissRequests: { [key: number]: Promise<Response> } = {}
+  private options: SchemaRegistryAPIClientOptions | undefined
+
+  public cache: Cache
+
+  constructor(
+    { auth, clientId, host, retry, agent }: SchemaRegistryAPIClientArgs,
+    options?: SchemaRegistryAPIClientOptions,
+  ) {
+    this.api = API({ auth, clientId, host, retry, agent })
+    this.cache = new Cache()
+    this.options = options
+  }
+
+  private isConfluentSchema(
+    schema: RawAvroSchema | AvroSchema | ConfluentSchema,
+  ): schema is ConfluentSchema {
+    return (schema as ConfluentSchema).schema != null
+  }
+
+  private getConfluentSchema(
+    schema: RawAvroSchema | AvroSchema | ConfluentSchema,
+  ): ConfluentSchema {
+    let confluentSchema: ConfluentSchema
+    // convert data from old api (for backwards compatibility)
+    if (!this.isConfluentSchema(schema)) {
+      // schema is instanceof RawAvroSchema or AvroSchema
+      confluentSchema = {
+        type: SchemaType.AVRO,
+        schema: JSON.stringify(schema),
+      }
+    } else {
+      confluentSchema = schema as ConfluentSchema
+    }
+    return confluentSchema
+  }
+
+  public async register(
+    schema: Exclude<ConfluentSchema, AvroConfluentSchema>,
+    userOpts: Opts,
+  ): Promise<RegisteredSchema>
+  public async register(
+    schema: RawAvroSchema | AvroConfluentSchema,
+    userOpts?: Omit<Opts, 'subject'> & { subject?: string },
+  ): Promise<RegisteredSchema>
+  public async register(
+    schema: RawAvroSchema | ConfluentSchema,
+    userOpts: Opts,
+  ): Promise<RegisteredSchema>
+  public async register(
+    schema: RawAvroSchema | ConfluentSchema,
+    userOpts?: Opts,
+  ): Promise<RegisteredSchema> {
+    const { compatibility, separator } = { ...DEFAULT_OPTS, ...userOpts }
+
+    const confluentSchema: ConfluentSchema = this.getConfluentSchema(schema)
+
+    const helper = helperTypeFromSchemaType(confluentSchema.type)
+
+    const options = await this.updateOptionsWithSchemaReferences(confluentSchema, this.options)
+    const schemaInstance = schemaFromConfluentSchema(confluentSchema, options)
+    helper.validate(schemaInstance)
+    let isFirstTimeRegistration = false
+    let subject: ConfluentSubject
+    if (userOpts?.subject) {
+      subject = {
+        name: userOpts.subject,
+      }
+    } else {
+      subject = helper.getSubject(confluentSchema, schemaInstance, separator)
+    }
+
+    try {
+      const response = await this.api.Subject.config({ subject: subject.name })
+      const { compatibilityLevel }: { compatibilityLevel: COMPATIBILITY } = response.data()
+
+      if (compatibilityLevel.toUpperCase() !== compatibility) {
+        throw new ConfluentSchemaRegistryCompatibilityError(
+          `Compatibility does not match the configuration (${compatibility} != ${compatibilityLevel.toUpperCase()})`,
+        )
+      }
+    } catch (error) {
+      if (error.status !== 404) {
+        throw error
+      } else {
+        isFirstTimeRegistration = true
+      }
+    }
+
+    const response = await this.api.Subject.register({
+      subject: subject.name,
+      body: {
+        schemaType: confluentSchema.type === SchemaType.AVRO ? undefined : confluentSchema.type,
+        schema: confluentSchema.schema,
+        references: confluentSchema.references,
+      },
+    })
+
+    if (compatibility && isFirstTimeRegistration) {
+      await this.api.Subject.updateConfig({ subject: subject.name, body: { compatibility } })
+    }
+
+    const registeredSchema: RegisteredSchema = response.data()
+    this.cache.setLatestRegistryId(subject.name, registeredSchema.id)
+    this.cache.setSchema(registeredSchema.id, confluentSchema.type, schemaInstance)
+
+    return registeredSchema
+  }
+
+  private async updateOptionsWithSchemaReferences(
+    schema: ConfluentSchema,
+    options?: SchemaRegistryAPIClientOptions,
+  ) {
+    const helper = helperTypeFromSchemaType(schema.type)
+    const referencedSchemas = await this.getreferencedSchemas(schema, helper)
+
+    const protocolOptions = this.asProtocolOptions(options)
+    return helper.updateOptionsFromSchemaReferences(referencedSchemas, protocolOptions)
+  }
+
+  private asProtocolOptions(options?: SchemaRegistryAPIClientOptions): ProtocolOptions | undefined {
+    if (!(options as LegacyOptions)?.forSchemaOptions) {
+      return options as ProtocolOptions | undefined
+    }
+    return {
+      [SchemaType.AVRO]: (options as LegacyOptions)?.forSchemaOptions,
+    }
+  }
+
+  private async getreferencedSchemas(
+    schema: ConfluentSchema,
+    helper: SchemaHelper,
+  ): Promise<ConfluentSchema[]> {
+    const referencesSet = new Set<string>()
+    return this.getreferencedSchemasRecursive(schema, helper, referencesSet)
+  }
+
+  private async getreferencedSchemasRecursive(
+    schema: ConfluentSchema,
+    helper: SchemaHelper,
+    referencesSet: Set<string>,
+  ): Promise<ConfluentSchema[]> {
+    const references = schema.references || []
+
+    let referencedSchemas: ConfluentSchema[] = []
+    for (const reference of references) {
+      const schemas = await this.getreferencedSchemasFromReference(reference, helper, referencesSet)
+      referencedSchemas = referencedSchemas.concat(schemas)
+    }
+    return referencedSchemas
+  }
+
+  async getreferencedSchemasFromReference(
+    reference: SchemaReference,
+    helper: SchemaHelper,
+    referencesSet: Set<string>,
+  ): Promise<ConfluentSchema[]> {
+    const { name, subject, version } = reference
+    const key = `${name}-${subject}-${version}`
+
+    // avoid duplicates
+    if (referencesSet.has(key)) {
+      return []
+    }
+    referencesSet.add(key)
+
+    const versionResponse = await this.api.Subject.version(reference)
+    const foundSchema = versionResponse.data() as SchemaResponse
+
+    const schema = helper.toConfluentSchema(foundSchema)
+    const referencedSchemas = await this.getreferencedSchemasRecursive(
+      schema,
+      helper,
+      referencesSet,
+    )
+
+    referencedSchemas.push(schema)
+    return referencedSchemas
+  }
+
+  private async _getSchema(
+    registryId: number,
+  ): Promise<{ type: SchemaType; schema: Schema | AvroSchema }> {
+    const cacheEntry = this.cache.getSchema(registryId)
+
+    if (cacheEntry) {
+      return cacheEntry
+    }
+
+    const response = await this.getSchemaOriginRequest(registryId)
+    const foundSchema: SchemaResponse = response.data()
+
+    const schemaType = schemaTypeFromString(foundSchema.schemaType)
+
+    const helper = helperTypeFromSchemaType(schemaType)
+    const confluentSchema = helper.toConfluentSchema(foundSchema)
+
+    const options = await this.updateOptionsWithSchemaReferences(confluentSchema, this.options)
+    const schemaInstance = schemaFromConfluentSchema(confluentSchema, options)
+    return this.cache.setSchema(registryId, schemaType, schemaInstance)
+  }
+
+  public async getSchema(registryId: number): Promise<Schema | AvroSchema> {
+    return await (await this._getSchema(registryId)).schema
+  }
+
+  public async encode(registryId: number, payload: any): Promise<Buffer> {
+    if (!registryId) {
+      throw new ConfluentSchemaRegistryArgumentError(
+        `Invalid registryId: ${JSON.stringify(registryId)}`,
+      )
+    }
+
+    const { schema } = await this._getSchema(registryId)
+    try {
+      const serializedPayload = schema.toBuffer(payload)
+      return encode(registryId, serializedPayload)
+    } catch (error) {
+      if (error instanceof ConfluentSchemaRegistryValidationError) throw error
+
+      const paths = this.collectInvalidPaths(schema, payload)
+      throw new ConfluentSchemaRegistryValidationError(error, paths)
+    }
+  }
+
+  private collectInvalidPaths(schema: Schema, jsonPayload: object) {
+    const paths: string[][] = []
+    schema.isValid(jsonPayload, {
+      errorHook: path => paths.push(path),
+    })
+
+    return paths
+  }
+
+  public async decode(buffer: Buffer, options?: DecodeOptions): Promise<any> {
+    if (!Buffer.isBuffer(buffer)) {
+      throw new ConfluentSchemaRegistryArgumentError('Invalid buffer')
+    }
+
+    const { magicByte, registryId, payload } = decode(buffer)
+    if (Buffer.compare(MAGIC_BYTE, magicByte) !== 0) {
+      throw new ConfluentSchemaRegistryArgumentError(
+        `Message encoded with magic byte ${JSON.stringify(magicByte)}, expected ${JSON.stringify(
+          MAGIC_BYTE,
+        )}`,
+      )
+    }
+
+    const { type, schema: writerSchema } = await this._getSchema(registryId)
+
+    let rawReaderSchema
+    switch (type) {
+      case SchemaType.AVRO:
+        rawReaderSchema = options?.[SchemaType.AVRO]?.readerSchema as RawAvroSchema | AvroSchema
+    }
+    if (rawReaderSchema) {
+      const readerSchema = schemaFromConfluentSchema(
+        { type: SchemaType.AVRO, schema: rawReaderSchema },
+        this.options,
+      ) as AvroSchema
+      if (readerSchema.equals(writerSchema as Type)) {
+        /* Even when schemas are considered equal by `avsc`,
+         * they still aren't interchangeable:
+         * provided `readerSchema` may have different `opts` (e.g. logicalTypes / unionWrap flags)
+         * see https://github.com/mtth/avsc/issues/362 */
+        return readerSchema.fromBuffer(payload)
+      } else {
+        // decode using a resolver from writer type into reader type
+        return readerSchema.fromBuffer(payload, readerSchema.createResolver(writerSchema as Type))
+      }
+    }
+
+    return writerSchema.fromBuffer(payload)
+  }
+
+  public async getRegistryId(subject: string, version: number | string): Promise<number> {
+    const response = await this.api.Subject.version({ subject, version })
+    const { id }: { id: number } = response.data()
+
+    return id
+  }
+
+  public async getRegistryIdBySchema(
+    subject: string,
+    schema: AvroSchema | RawAvroSchema | ConfluentSchema,
+  ): Promise<number> {
+    try {
+      const confluentSchema: ConfluentSchema = this.getConfluentSchema(schema)
+      const response = await this.api.Subject.registered({
+        subject,
+        body: {
+          schemaType: confluentSchema.type === SchemaType.AVRO ? undefined : confluentSchema.type,
+          schema: confluentSchema.schema,
+        },
+      })
+      const { id }: { id: number } = response.data()
+
+      return id
+    } catch (error) {
+      if (error.status && error.status === 404) {
+        throw new ConfluentSchemaRegistryError(error)
+      }
+
+      throw error
+    }
+  }
+
+  public async getLatestSchemaId(subject: string): Promise<number> {
+    const response = await this.api.Subject.latestVersion({ subject })
+    const { id }: { id: number } = response.data()
+
+    return id
+  }
+
+  private getSchemaOriginRequest(registryId: number) {
+    // ensure that cache-misses result in a single origin request
+    if (this.cacheMissRequests[registryId]) {
+      return this.cacheMissRequests[registryId]
+    } else {
+      const request = this.api.Schema.find({ id: registryId }).finally(() => {
+        delete this.cacheMissRequests[registryId]
+      })
+
+      this.cacheMissRequests[registryId] = request
+
+      return request
+    }
+  }
+}

package/src/api/index.spec.ts

@@ -0,0 +1,23 @@
+import API from '.'
+import { mockClient, install, uninstall } from 'mappersmith/test'
+
+const client = API({ clientId: 'test-client', host: 'http://example.com' })
+const mock = mockClient<typeof client>(client)
+  .resource('Schema')
+  .method('find')
+  .with({ id: 'abc' })
+  .response({})
+  .assertObject()
+
+describe('API Client', () => {
+  beforeEach(() => install())
+
+  afterEach(() => uninstall())
+
+  it('should include a user agent header', async () => {
+    const response = await client.Schema.find({ id: 'abc' })
+
+    expect(mock.callsCount()).toBe(1)
+    expect(response.request().header('User-Agent')).not.toBeUndefined()
+  })
+})

package/src/api/index.ts

@@ -0,0 +1,121 @@
+import { Agent } from 'http'
+import forge, { Authorization, Client, Options, GatewayConfiguration } from 'mappersmith'
+import RetryMiddleware, { RetryMiddlewareOptions } from 'mappersmith/middleware/retry/v2'
+import BasicAuthMiddleware from 'mappersmith/middleware/basic-auth'
+
+import { DEFAULT_API_CLIENT_ID } from '../constants'
+import errorMiddleware from './middleware/errorMiddleware'
+import confluentEncoder from './middleware/confluentEncoderMiddleware'
+import userAgentMiddleware from './middleware/userAgent'
+
+const DEFAULT_RETRY = {
+  maxRetryTimeInSecs: 5,
+  initialRetryTimeInSecs: 0.1,
+  factor: 0.2, // randomization factor
+  multiplier: 2, // exponential factor
+  retries: 3, // max retries
+}
+
+export interface SchemaRegistryAPIClientArgs {
+  host: string
+  auth?: Authorization
+  clientId?: string
+  retry?: Partial<RetryMiddlewareOptions>
+  /** HTTP Agent that will be passed to underlying API calls */
+  agent?: Agent
+}
+
+// TODO: Improve typings
+export type SchemaRegistryAPIClient = Client<{
+  Schema: {
+    find: (_: any) => any
+  }
+  Subject: {
+    all: (_: any) => any
+    latestVersion: (_: any) => any
+    version: (_: any) => any
+    config: (_: any) => any
+    updateConfig: (_: any) => any
+    register: (_: any) => any
+    registered: (_: any) => any
+    compatible: (_: any) => any
+  }
+}>
+
+export default ({
+  auth,
+  clientId: userClientId,
+  host,
+  retry = {},
+  agent,
+}: SchemaRegistryAPIClientArgs): SchemaRegistryAPIClient => {
+  const clientId = userClientId || DEFAULT_API_CLIENT_ID
+  // FIXME: ResourcesType typings is not exposed by mappersmith
+  const manifest: Options<any> = {
+    clientId,
+    ignoreGlobalMiddleware: true,
+    host,
+    middleware: [
+      userAgentMiddleware,
+      confluentEncoder,
+      RetryMiddleware(Object.assign(DEFAULT_RETRY, retry)),
+      errorMiddleware,
+      ...(auth ? [BasicAuthMiddleware(auth)] : []),
+    ],
+    resources: {
+      Schema: {
+        find: {
+          method: 'get',
+          path: '/schemas/ids/{id}',
+        },
+      },
+      Subject: {
+        all: {
+          method: 'get',
+          path: '/subjects',
+        },
+        latestVersion: {
+          method: 'get',
+          path: '/subjects/{subject}/versions/latest',
+        },
+        version: {
+          method: 'get',
+          path: '/subjects/{subject}/versions/{version}',
+        },
+        registered: {
+          method: 'post',
+          path: '/subjects/{subject}',
+        },
+
+        config: {
+          method: 'get',
+          path: '/config/{subject}',
+        },
+        updateConfig: {
+          method: 'put',
+          path: '/config/{subject}',
+        },
+
+        register: {
+          method: 'post',
+          path: '/subjects/{subject}/versions',
+        },
+        compatible: {
+          method: 'post',
+          path: '/compatibility/subjects/{subject}/versions/{version}',
+          params: { version: 'latest' },
+        },
+      },
+    },
+  }
+  // if an agent was provided, bind the agent to the mappersmith configs
+  if (agent) {
+    // gatewayConfigs is not listed as a type on manifest object in mappersmith
+    ;((manifest as unknown) as { gatewayConfigs: Partial<GatewayConfiguration> }).gatewayConfigs = {
+      HTTP: {
+        configure: () => ({ agent }),
+      },
+    }
+  }
+  return forge(manifest)
+}

package/src/api/middleware/confluentEncoderMiddleware.ts

@@ -0,0 +1,36 @@
+import { Middleware, Response } from 'mappersmith'
+
+const REQUEST_HEADERS = {
+  'Content-Type': 'application/vnd.schemaregistry.v1+json',
+}
+
+const updateContentType = (response: Response) =>
+  response.enhance({
+    headers: {
+      'content-type': 'application/json',
+    },
+  })
+
+const confluentEncoderMiddleware: Middleware = () => ({
+  request: req => {
+    try {
+      if (req.body()) {
+        return req.enhance({
+          headers: REQUEST_HEADERS,
+          body: JSON.stringify(req.body()),
+        })
+      }
+    } catch (_) {}
+
+    return req.enhance({ headers: REQUEST_HEADERS })
+  },
+
+  response: next =>
+    next()
+      .then(updateContentType)
+      .catch((response: Response) => {
+        throw updateContentType(response)
+      }),
+})
+
+export default confluentEncoderMiddleware

package/src/api/middleware/errorMiddleware.spec.ts

@@ -0,0 +1,67 @@
+import { MiddlewareDescriptor } from 'mappersmith'
+
+import ErrorMiddleware from './errorMiddleware'
+
+const middlewareParams = {
+  resourceName: 'resourceNameMock',
+  resourceMethod: 'resourceMethodMock',
+  context: { context: 'contextMock' },
+  clientId: 'clientIdMock',
+}
+
+describe('ErrorMiddleware', () => {
+  let executedMiddleware: MiddlewareDescriptor
+
+  beforeEach(() => {
+    executedMiddleware = ErrorMiddleware(middlewareParams)
+  })
+
+  describe('when the request succeeds', () => {
+    it('does not interfere with the promise', async () => {
+      await expect(
+        // @ts-ignore
+        executedMiddleware.response(() => Promise.resolve('arbitrary value'), undefined),
+      ).resolves.toBe('arbitrary value')
+    })
+  })
+
+  describe('when the request fails', () => {
+    const createResponse = data => ({
+      data: jest.fn(() => data),
+      status: jest.fn(() => 500),
+      request: jest.fn(() => ({
+        method: jest.fn(() => 'get'),
+        url: jest.fn(() => 'url'),
+      })),
+    })
+
+    it('raise an error with message', async () => {
+      const message = 'error message'
+      const response = createResponse({ message })
+
+      await expect(
+        executedMiddleware.response(() => Promise.reject(response), undefined),
+      ).rejects.toHaveProperty('message', `${middlewareParams.clientId} - ${message}`)
+    })
+
+    it('raises an error with a message in case of client-side errors', async () => {
+      const message = 'error message'
+      const response = createResponse(message)
+
+      await expect(
+        executedMiddleware.response(() => Promise.reject(response), undefined),
+      ).rejects.toHaveProperty(
+        'message',
+        `${middlewareParams.clientId} - Error, status 500: ${message}`,
+      )
+    })
+
+    it('raise an error with a default message if the error payload is empty', async () => {
+      const response = createResponse('')
+
+      await expect(
+        executedMiddleware.response(() => Promise.reject(response), undefined),
+      ).rejects.toHaveProperty('message', `${middlewareParams.clientId} - Error, status 500`)
+    })
+  })
+})

package/src/api/middleware/errorMiddleware.ts

@@ -0,0 +1,37 @@
+import { Middleware, Response } from 'mappersmith'
+
+interface ConfluenceResponse extends Omit<Response, 'data'> {
+  data: () => {
+    message: string
+  }
+}
+
+class ResponseError extends Error {
+  status: number
+  unauthorized: boolean
+  url: string
+
+  constructor(clientName: string, response: ConfluenceResponse) {
+    super(
+      `${clientName} - ${response.data().message ||
+        `Error, status ${response.status()}${response.data() ? `: ${response.data()}` : ''}`}`,
+    )
+
+    const request = response.request()
+    this.name = this.constructor.name
+    this.status = response.status()
+    this.unauthorized = this.status === 401
+    this.url = `${request.method()} ${request.url()}`
+  }
+}
+
+const errorMiddleware: Middleware = ({ clientId }) => ({
+  response: next =>
+    new Promise((resolve, reject) =>
+      next()
+        .then(resolve)
+        .catch((response: Response) => reject(new ResponseError(clientId, response))),
+    ),
+})
+
+export default errorMiddleware