@llc3233149/core 0.9.0

package/src/kiwi/kiwi-schema/bb.ts

@@ -0,0 +1,244 @@
+let int32 = new Int32Array(1)
+let float32 = new Float32Array(int32.buffer)
+const textDecoder = new TextDecoder()
+
+export class ByteBuffer {
+  private _data: Uint8Array
+  private _index: number
+  length: number
+
+  constructor(data?: Uint8Array) {
+    if (data && !(data instanceof Uint8Array)) {
+      throw new Error('Must initialize a ByteBuffer with a Uint8Array')
+    }
+    this._data = data || new Uint8Array(256)
+    this._index = 0
+    this.length = data ? data.length : 0
+  }
+
+  toUint8Array(): Uint8Array {
+    return this._data.subarray(0, this.length)
+  }
+
+  readByte(): number {
+    return this._data[this._index++]
+  }
+
+  readByteArray(): Uint8Array {
+    const length = this.readVarUint()
+    const start = this._index
+    this._index = start + length
+    return this._data.slice(start, start + length)
+  }
+
+  readVarFloat(): number {
+    const index = this._index
+    const data = this._data
+    const first = data[index]
+    if (first === 0) {
+      this._index = index + 1
+      return 0
+    }
+
+    let bits = first | (data[index + 1] << 8) | (data[index + 2] << 16) | (data[index + 3] << 24)
+    this._index = index + 4
+    bits = (bits << 23) | (bits >>> 9)
+    int32[0] = bits
+    return float32[0]
+  }
+
+  readVarUint(): number {
+    const data = this._data
+    let i = this._index
+    let b = data[i++]
+    let value = b & 127
+    if (b < 128) {
+      this._index = i
+      return value
+    }
+    b = data[i++]
+    value |= (b & 127) << 7
+    if (b < 128) {
+      this._index = i
+      return value
+    }
+    b = data[i++]
+    value |= (b & 127) << 14
+    if (b < 128) {
+      this._index = i
+      return value
+    }
+    b = data[i++]
+    value |= (b & 127) << 21
+    if (b < 128) {
+      this._index = i
+      return value
+    }
+    b = data[i++]
+    value |= (b & 127) << 28
+    this._index = i
+    return value >>> 0
+  }
+
+  readVarInt(): number {
+    let value = this.readVarUint() | 0
+    return value & 1 ? ~(value >>> 1) : value >>> 1
+  }
+
+  readVarUint64(): bigint {
+    let value = BigInt(0)
+    let shift = BigInt(0)
+    let seven = BigInt(7)
+    let byte: number
+    while ((byte = this.readByte()) & 128 && shift < 56) {
+      value |= BigInt(byte & 127) << shift
+      shift += seven
+    }
+    value |= BigInt(byte) << shift
+    return value
+  }
+
+  readVarInt64(): bigint {
+    let value = this.readVarUint64()
+    let one = BigInt(1)
+    let sign = value & one
+    value >>= one
+    return sign ? ~value : value
+  }
+
+  readString(): string {
+    const data = this._data
+    const start = this._index
+    let i = start
+    while (data[i] !== 0) i++
+    this._index = i + 1
+    return textDecoder.decode(data.subarray(start, i))
+  }
+
+  private _growBy(amount: number): void {
+    if (this.length + amount > this._data.length) {
+      let data = new Uint8Array((this.length + amount) << 1)
+      data.set(this._data)
+      this._data = data
+    }
+    this.length += amount
+  }
+
+  writeByte(value: number): void {
+    let index = this.length
+    this._growBy(1)
+    this._data[index] = value
+  }
+
+  writeByteArray(value: Uint8Array): void {
+    this.writeVarUint(value.length)
+    let index = this.length
+    this._growBy(value.length)
+    this._data.set(value, index)
+  }
+
+  writeVarFloat(value: number): void {
+    let index = this.length
+
+    // Reinterpret as an integer
+    float32[0] = value
+    let bits = int32[0]
+
+    // Move the exponent to the first 8 bits
+    bits = (bits >>> 23) | (bits << 9)
+
+    // Optimization: use a single byte to store zero and denormals (check for an exponent of 0)
+    if ((bits & 255) === 0) {
+      this.writeByte(0)
+      return
+    }
+
+    // Endian-independent 32-bit write
+    this._growBy(4)
+    let data = this._data
+    data[index] = bits
+    data[index + 1] = bits >> 8
+    data[index + 2] = bits >> 16
+    data[index + 3] = bits >> 24
+  }
+
+  writeVarUint(value: number): void {
+    if (value < 0 || value > 0xffff_ffff) throw new Error('Outside uint range: ' + value)
+    do {
+      let byte = value & 127
+      value >>>= 7
+      this.writeByte(value ? byte | 128 : byte)
+    } while (value)
+  }
+
+  writeVarInt(value: number): void {
+    if (value < -0x8000_0000 || value > 0x7fff_ffff) throw new Error('Outside int range: ' + value)
+    this.writeVarUint(((value << 1) ^ (value >> 31)) >>> 0)
+  }
+
+  writeVarUint64(value: bigint | string): void {
+    if (typeof value === 'string') value = BigInt(value)
+    else if (typeof value !== 'bigint')
+      throw new Error('Expected bigint but got ' + typeof value + ': ' + value)
+    if (value < 0 || value > BigInt('0xFFFFFFFFFFFFFFFF'))
+      throw new Error('Outside uint64 range: ' + value)
+    let mask = BigInt(127)
+    let seven = BigInt(7)
+    for (let i = 0; value > mask && i < 8; i++) {
+      this.writeByte(Number(value & mask) | 128)
+      value >>= seven
+    }
+    this.writeByte(Number(value))
+  }
+
+  writeVarInt64(value: bigint | string): void {
+    if (typeof value === 'string') value = BigInt(value)
+    else if (typeof value !== 'bigint')
+      throw new Error('Expected bigint but got ' + typeof value + ': ' + value)
+    if (value < -BigInt('0x8000000000000000') || value > BigInt('0x7FFFFFFFFFFFFFFF'))
+      throw new Error('Outside int64 range: ' + value)
+    let one = BigInt(1)
+    this.writeVarUint64(value < 0 ? ~(value << one) : value << one)
+  }
+
+  writeString(value: string): void {
+    let codePoint
+
+    for (let i = 0; i < value.length; i++) {
+      // Decode UTF-16
+      let a = value.charCodeAt(i)
+      if (i + 1 === value.length || a < 0xd800 || a >= 0xdc00) {
+        codePoint = a
+      } else {
+        let b = value.charCodeAt(++i)
+        codePoint = (a << 10) + b + (0x10000 - (0xd800 << 10) - 0xdc00)
+      }
+
+      // Strings are null-terminated
+      if (codePoint === 0) {
+        throw new Error('Cannot encode a string containing the null character')
+      }
+
+      // Encode UTF-8
+      if (codePoint < 0x80) {
+        this.writeByte(codePoint)
+      } else {
+        if (codePoint < 0x800) {
+          this.writeByte(((codePoint >> 6) & 0x1f) | 0xc0)
+        } else {
+          if (codePoint < 0x10000) {
+            this.writeByte(((codePoint >> 12) & 0x0f) | 0xe0)
+          } else {
+            this.writeByte(((codePoint >> 18) & 0x07) | 0xf0)
+            this.writeByte(((codePoint >> 12) & 0x3f) | 0x80)
+          }
+          this.writeByte(((codePoint >> 6) & 0x3f) | 0x80)
+        }
+        this.writeByte((codePoint & 0x3f) | 0x80)
+      }
+    }
+
+    // Strings are null-terminated
+    this.writeByte(0)
+  }
+}

package/src/kiwi/kiwi-schema/binary.ts

@@ -0,0 +1,102 @@
+import { ByteBuffer } from './bb'
+import { Schema, Field, Definition, DefinitionKind } from './schema'
+
+let types: (string | null)[] = ['bool', 'byte', 'int', 'uint', 'float', 'string', 'int64', 'uint64']
+let kinds: DefinitionKind[] = ['ENUM', 'STRUCT', 'MESSAGE']
+
+export function decodeBinarySchema(buffer: Uint8Array | ByteBuffer): Schema {
+  let bb = buffer instanceof ByteBuffer ? buffer : new ByteBuffer(buffer)
+  let definitionCount = bb.readVarUint()
+  let definitions: Definition[] = []
+
+  // Read in the schema
+  for (let i = 0; i < definitionCount; i++) {
+    let definitionName = bb.readString()
+    let kind = bb.readByte()
+    let fieldCount = bb.readVarUint()
+    let fields: Field[] = []
+
+    for (let j = 0; j < fieldCount; j++) {
+      let fieldName = bb.readString()
+      let type = bb.readVarInt()
+      let isArray = !!(bb.readByte() & 1)
+      let value = bb.readVarUint()
+
+      fields.push({
+        name: fieldName,
+        line: 0,
+        column: 0,
+        type: kinds[kind] === 'ENUM' ? null : (type as any),
+        isArray: isArray,
+        isDeprecated: false,
+        value: value
+      })
+    }
+
+    definitions.push({
+      name: definitionName,
+      line: 0,
+      column: 0,
+      kind: kinds[kind],
+      fields: fields
+    })
+  }
+
+  // Bind type names afterwards
+  for (let i = 0; i < definitionCount; i++) {
+    let fields = definitions[i].fields
+    for (let j = 0; j < fields.length; j++) {
+      let field = fields[j]
+      let type = field.type as any as number | null
+
+      if (type !== null && type < 0) {
+        if (~type >= types.length) {
+          throw new Error('Invalid type ' + type)
+        }
+        field.type = types[~type]
+      } else {
+        if (type !== null && type >= definitions.length) {
+          throw new Error('Invalid type ' + type)
+        }
+        field.type = type === null ? null : definitions[type].name
+      }
+    }
+  }
+
+  return {
+    package: null,
+    definitions: definitions
+  }
+}
+
+export function encodeBinarySchema(schema: Schema): Uint8Array {
+  let bb = new ByteBuffer()
+  let definitions = schema.definitions
+  let definitionIndex: { [name: string]: number } = {}
+
+  bb.writeVarUint(definitions.length)
+
+  for (let i = 0; i < definitions.length; i++) {
+    definitionIndex[definitions[i].name] = i
+  }
+
+  for (let i = 0; i < definitions.length; i++) {
+    let definition = definitions[i]
+
+    bb.writeString(definition.name)
+    bb.writeByte(kinds.indexOf(definition.kind))
+    bb.writeVarUint(definition.fields.length)
+
+    for (let j = 0; j < definition.fields.length; j++) {
+      let field = definition.fields[j]
+      let type = types.indexOf(field.type)
+
+      bb.writeString(field.name)
+      bb.writeVarInt(type === -1 ? definitionIndex[field.type!] : ~type)
+      bb.writeByte(field.isArray ? 1 : 0)
+      bb.writeVarUint(field.value)
+    }
+  }
+
+  return bb.toUint8Array()
+}

package/src/kiwi/kiwi-schema/index.ts

@@ -0,0 +1,5 @@
+export type { Schema, Definition, Field } from './schema'
+export { ByteBuffer } from './bb'
+export { compileSchema } from './js'
+export { decodeBinarySchema, encodeBinarySchema } from './binary'
+export { parseSchema } from './parser'

package/src/kiwi/kiwi-schema/js.ts

@@ -0,0 +1,339 @@
+import { ByteBuffer } from './bb'
+import { Schema, Definition } from './schema'
+import { error, quote } from './util'
+
+function compileDecode(
+  definition: Definition,
+  definitions: { [name: string]: Definition }
+): string {
+  let lines: string[] = []
+  let indent = '  '
+
+  lines.push('function (bb) {')
+  lines.push('  var result = {};')
+  lines.push('  if (!(bb instanceof this.ByteBuffer)) {')
+  lines.push('    bb = new this.ByteBuffer(bb);')
+  lines.push('  }')
+  lines.push('')
+
+  if (definition.kind === 'MESSAGE') {
+    lines.push('  while (true) {')
+    lines.push('    switch (bb.readVarUint()) {')
+    lines.push('      case 0:')
+    lines.push('        return result;')
+    lines.push('')
+    indent = '        '
+  }
+
+  for (let i = 0; i < definition.fields.length; i++) {
+    let field = definition.fields[i]
+    let code: string
+
+    switch (field.type) {
+      case 'bool': {
+        code = '!!bb.readByte()'
+        break
+      }
+
+      case 'byte': {
+        code = 'bb.readByte()' // only used if not array
+        break
+      }
+
+      case 'int': {
+        code = 'bb.readVarInt()'
+        break
+      }
+
+      case 'uint': {
+        code = 'bb.readVarUint()'
+        break
+      }
+
+      case 'float': {
+        code = 'bb.readVarFloat()'
+        break
+      }
+
+      case 'string': {
+        code = 'bb.readString()'
+        break
+      }
+
+      case 'int64': {
+        code = 'bb.readVarInt64()'
+        break
+      }
+
+      case 'uint64': {
+        code = 'bb.readVarUint64()'
+        break
+      }
+
+      default: {
+        let type = definitions[field.type!]
+        if (!type) {
+          error(
+            'Invalid type ' + quote(field.type!) + ' for field ' + quote(field.name),
+            field.line,
+            field.column
+          )
+        } else if (type.kind === 'ENUM') {
+          code = 'this[' + quote(type.name) + '][bb.readVarUint()]'
+        } else {
+          code = 'this[' + quote('decode' + type.name) + '](bb)'
+        }
+      }
+    }
+
+    if (definition.kind === 'MESSAGE') {
+      lines.push('      case ' + field.value + ':')
+    }
+
+    if (field.isArray) {
+      if (field.isDeprecated) {
+        if (field.type === 'byte') {
+          lines.push(indent + 'bb.readByteArray();')
+        } else {
+          lines.push(indent + 'var length = bb.readVarUint();')
+          lines.push(indent + 'while (length-- > 0) ' + code + ';')
+        }
+      } else {
+        if (field.type === 'byte') {
+          lines.push(indent + 'result[' + quote(field.name) + '] = bb.readByteArray();')
+        } else {
+          lines.push(indent + 'var length = bb.readVarUint();')
+          lines.push(indent + 'var values = result[' + quote(field.name) + '] = Array(length);')
+          lines.push(indent + 'for (var i = 0; i < length; i++) values[i] = ' + code + ';')
+        }
+      }
+    } else {
+      if (field.isDeprecated) {
+        lines.push(indent + code + ';')
+      } else {
+        lines.push(indent + 'result[' + quote(field.name) + '] = ' + code + ';')
+      }
+    }
+
+    if (definition.kind === 'MESSAGE') {
+      lines.push('        break;')
+      lines.push('')
+    }
+  }
+
+  if (definition.kind === 'MESSAGE') {
+    lines.push('      default:')
+    lines.push('        throw new Error("Attempted to parse invalid message");')
+    lines.push('    }')
+    lines.push('  }')
+  } else {
+    lines.push('  return result;')
+  }
+
+  lines.push('}')
+
+  return lines.join('\n')
+}
+
+function compileEncode(
+  definition: Definition,
+  definitions: { [name: string]: Definition }
+): string {
+  let lines: string[] = []
+
+  lines.push('function (message, bb) {')
+  lines.push('  var isTopLevel = !bb;')
+  lines.push('  if (isTopLevel) bb = new this.ByteBuffer();')
+
+  for (let j = 0; j < definition.fields.length; j++) {
+    let field = definition.fields[j]
+    let code: string
+
+    if (field.isDeprecated) {
+      continue
+    }
+
+    switch (field.type) {
+      case 'bool': {
+        code = 'bb.writeByte(value);'
+        break
+      }
+
+      case 'byte': {
+        code = 'bb.writeByte(value);' // only used if not array
+        break
+      }
+
+      case 'int': {
+        code = 'bb.writeVarInt(value);'
+        break
+      }
+
+      case 'uint': {
+        code = 'bb.writeVarUint(value);'
+        break
+      }
+
+      case 'float': {
+        code = 'bb.writeVarFloat(value);'
+        break
+      }
+
+      case 'string': {
+        code = 'bb.writeString(value);'
+        break
+      }
+
+      case 'int64': {
+        code = 'bb.writeVarInt64(value);'
+        break
+      }
+
+      case 'uint64': {
+        code = 'bb.writeVarUint64(value);'
+        break
+      }
+
+      default: {
+        let type = definitions[field.type!]
+        if (!type) {
+          throw new Error('Invalid type ' + quote(field.type!) + ' for field ' + quote(field.name))
+        } else if (type.kind === 'ENUM') {
+          code =
+            'var encoded = this[' +
+            quote(type.name) +
+            '][value]; ' +
+            'if (encoded === void 0) throw new Error("Invalid value " + JSON.stringify(value) + ' +
+            quote(' for enum ' + quote(type.name)) +
+            '); ' +
+            'bb.writeVarUint(encoded);'
+        } else {
+          code = 'this[' + quote('encode' + type.name) + '](value, bb);'
+        }
+      }
+    }
+
+    lines.push('')
+    lines.push('  var value = message[' + quote(field.name) + '];')
+    lines.push('  if (value != null) {') // Comparing with null using "!=" also checks for undefined
+
+    if (definition.kind === 'MESSAGE') {
+      lines.push('    bb.writeVarUint(' + field.value + ');')
+    }
+
+    if (field.isArray) {
+      if (field.type === 'byte') {
+        lines.push('    bb.writeByteArray(value);')
+      } else {
+        lines.push('    var values = value, n = values.length;')
+        lines.push('    bb.writeVarUint(n);')
+        lines.push('    for (var i = 0; i < n; i++) {')
+        lines.push('      value = values[i];')
+        lines.push('      ' + code)
+        lines.push('    }')
+      }
+    } else {
+      lines.push('    ' + code)
+    }
+
+    if (definition.kind === 'STRUCT') {
+      lines.push('  } else {')
+      lines.push(
+        '    throw new Error(' + quote('Missing required field ' + quote(field.name)) + ');'
+      )
+    }
+
+    lines.push('  }')
+  }
+
+  // A field id of zero is reserved to indicate the end of the message
+  if (definition.kind === 'MESSAGE') {
+    lines.push('  bb.writeVarUint(0);')
+  }
+
+  lines.push('')
+  lines.push('  if (isTopLevel) return bb.toUint8Array();')
+  lines.push('}')
+
+  return lines.join('\n')
+}
+
+export function compileSchemaJS(schema: Schema): string {
+  let definitions: { [name: string]: Definition } = {}
+  let name = schema.package
+  let js: string[] = []
+
+  if (name !== null) {
+    js.push('var ' + name + ' = exports || ' + name + ' || {}, exports;')
+  } else {
+    js.push('var exports = exports || {};')
+    name = 'exports'
+  }
+
+  js.push(name + '.ByteBuffer = ' + name + '.ByteBuffer || require("kiwi-schema").ByteBuffer;')
+
+  for (let i = 0; i < schema.definitions.length; i++) {
+    let definition = schema.definitions[i]
+    definitions[definition.name] = definition
+  }
+
+  for (let i = 0; i < schema.definitions.length; i++) {
+    let definition = schema.definitions[i]
+
+    switch (definition.kind) {
+      case 'ENUM': {
+        let value: any = {}
+        for (let j = 0; j < definition.fields.length; j++) {
+          let field = definition.fields[j]
+          value[field.name] = field.value
+          value[field.value] = field.name
+        }
+        js.push(name + '[' + quote(definition.name) + '] = ' + JSON.stringify(value, null, 2) + ';')
+        break
+      }
+
+      case 'STRUCT':
+      case 'MESSAGE': {
+        js.push('')
+        js.push(
+          name +
+            '[' +
+            quote('decode' + definition.name) +
+            '] = ' +
+            compileDecode(definition, definitions) +
+            ';'
+        )
+        js.push('')
+        js.push(
+          name +
+            '[' +
+            quote('encode' + definition.name) +
+            '] = ' +
+            compileEncode(definition, definitions) +
+            ';'
+        )
+        break
+      }
+
+      default: {
+        error(
+          'Invalid definition kind ' + quote(definition.kind),
+          definition.line,
+          definition.column
+        )
+        break
+      }
+    }
+  }
+
+  js.push('')
+  return js.join('\n')
+}
+
+export function compileSchema(schema: Schema): any {
+  let result = {
+    ByteBuffer: ByteBuffer
+  }
+  new Function('exports', compileSchemaJS(schema))(result)
+  return result
+}