goscript 0.1.4 → 0.2.0

package/gs/mime/index.test.ts

@@ -0,0 +1,90 @@
+import { describe, expect, it } from 'vitest'
+
+import {
+  AddExtensionType,
+  BEncoding,
+  ErrInvalidMediaParameter,
+  ExtensionsByType,
+  FormatMediaType,
+  ParseMediaType,
+  QEncoding,
+  TypeByExtension,
+  WordDecoder,
+  WordEncoder_Encode,
+} from './index.js'
+
+describe('mime override', () => {
+  it('parses, formats, and looks up media types', () => {
+    const [mediaType, params, err] = ParseMediaType(
+      'Text/Plain; Charset="utf-8"; format=flowed',
+    )
+    expect(err).toBeNull()
+    expect(mediaType).toBe('text/plain')
+    expect(params.get('charset')).toBe('utf-8')
+    expect(params.get('format')).toBe('flowed')
+    const [trailingMediaType, trailingParams, trailingErr] = ParseMediaType('text/plain;')
+    expect(trailingErr).toBeNull()
+    expect(trailingMediaType).toBe('text/plain')
+    expect(trailingParams.size).toBe(0)
+
+    expect(
+      FormatMediaType('text/plain', new Map([['charset', 'utf-8']])),
+    ).toBe('text/plain; charset=utf-8')
+    expect(
+      FormatMediaType('Attachment', new Map([['FileName', 'hello.txt']])),
+    ).toBe('attachment; filename=hello.txt')
+    expect(
+      FormatMediaType('attachment', new Map([['filename', 'foo bar.txt']])),
+    ).toBe('attachment; filename="foo bar.txt"')
+    expect(
+      FormatMediaType('text/plain', new Map([
+        ['z', 'last'],
+        ['a', 'first'],
+      ])),
+    ).toBe('text/plain; a=first; z=last')
+
+    const encoded = FormatMediaType(
+      'attachment',
+      new Map([['filename', 'résumé.txt']]),
+    )
+    expect(encoded).toBe("attachment; filename*=utf-8''r%C3%A9sum%C3%A9.txt")
+    const [, encodedParams, encodedErr] = ParseMediaType(encoded)
+    expect(encodedErr).toBeNull()
+    expect(encodedParams.get('filename')).toBe('résumé.txt')
+    const [, pathParams, pathErr] = ParseMediaType('form-data; filename="C:\\dev\\go.txt"; quoted="a\\"b"')
+    expect(pathErr).toBeNull()
+    expect(pathParams.get('filename')).toBe('C:\\dev\\go.txt')
+    expect(pathParams.get('quoted')).toBe('a"b')
+
+    expect(TypeByExtension('.json')).toBe('application/json')
+    expect(AddExtensionType('.goscript', 'text/plain; charset=utf-8')).toBeNull()
+    const [extensions, extensionErr] = ExtensionsByType('text/plain')
+    expect(extensionErr).toBeNull()
+    expect(extensions).toContain('.goscript')
+  })
+
+  it('reports invalid media parameters', () => {
+    const [, , err] = ParseMediaType('text/plain; broken')
+    expect(err).toBe(ErrInvalidMediaParameter)
+    expect(ParseMediaType('text/plain; filename=foo bar')[2]).toBe(ErrInvalidMediaParameter)
+    expect(ParseMediaType('text/plain; filename="unterminated')[2]).toBe(ErrInvalidMediaParameter)
+    expect(ParseMediaType('text/plain; x=1; x=2')[2]).toBe(ErrInvalidMediaParameter)
+    expect(ParseMediaType('text/plain; bad[]=x')[2]).toBe(ErrInvalidMediaParameter)
+    expect(ParseMediaType('text/plain; bad{}=x')[1].get('bad{}')).toBe('x')
+  })
+
+  it('encodes and decodes RFC 2047 words', () => {
+    expect(WordEncoder_Encode(BEncoding, 'utf-8', 'hello')).toBe('hello')
+
+    const bWord = WordEncoder_Encode(BEncoding, 'utf-8', 'héllo')
+    expect(new WordDecoder().Decode(bWord)).toEqual(['héllo', null])
+
+    const qWord = WordEncoder_Encode(QEncoding, 'utf-8', 'hello world')
+    expect(qWord).toBe('hello world')
+
+    const encodedQWord = WordEncoder_Encode(QEncoding, 'utf-8', 'héllo world')
+    const [header, err] = new WordDecoder().DecodeHeader(`subject ${encodedQWord}`)
+    expect(err).toBeNull()
+    expect(header).toBe('subject héllo world')
+  })
+})

package/gs/mime/index.ts

@@ -1,34 +1,253 @@
+import * as $ from '@goscript/builtin/index.js'
+import * as errors from '@goscript/errors/index.js'
+import type * as io from '@goscript/io/index.js'
+
+export type WordEncoder = number
+
+export const BEncoding: WordEncoder = 'b'.charCodeAt(0)
+export const QEncoding: WordEncoder = 'q'.charCodeAt(0)
+
+export const ErrInvalidMediaParameter = errors.New(
+  'mime: invalid media parameter',
+)
+
+const extensionTypes = new Map<string, string>([
+  ['.css', 'text/css; charset=utf-8'],
+  ['.gif', 'image/gif'],
+  ['.html', 'text/html; charset=utf-8'],
+  ['.htm', 'text/html; charset=utf-8'],
+  ['.jpeg', 'image/jpeg'],
+  ['.jpg', 'image/jpeg'],
+  ['.js', 'text/javascript; charset=utf-8'],
+  ['.json', 'application/json'],
+  ['.mjs', 'text/javascript; charset=utf-8'],
+  ['.png', 'image/png'],
+  ['.svg', 'image/svg+xml'],
+  ['.txt', 'text/plain; charset=utf-8'],
+  ['.wasm', 'application/wasm'],
+  ['.xml', 'text/xml; charset=utf-8'],
+])
+
+type nodeBufferConstructor = {
+  from(data: Uint8Array): { toString(encoding: string): string }
+  from(data: string, encoding: string): Uint8Array
+}
+
+type nodeProcess = {
+  getBuiltinModule?: (
+    name: string,
+  ) => { Buffer?: nodeBufferConstructor } | undefined
+}
+
 export function FormatMediaType(
   t: string,
   param: Map<string, string> | Record<string, string> | null,
 ): string {
-  if (!isToken(t)) {
+  const mediaType = formatMediaTypeDisposition(t)
+  if (mediaType === '') {
     return ''
   }
-  let out = t
+  let out = mediaType
   const entries =
     param instanceof Map ?
       Array.from(param.entries())
     : Object.entries(param ?? {})
-  entries.sort(([a], [b]) => a.localeCompare(b))
+  entries.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
   for (const [key, value] of entries) {
     if (!isToken(key)) {
       return ''
     }
-    out += formatParam(key, String(value))
+    out += formatParam(key.toLowerCase(), String(value))
   }
   return out
 }
 
+export function ParseMediaType(
+  v: string,
+): [string, Map<string, string>, $.GoError] {
+  const parts = splitMediaType(v)
+  const mediatype = formatMediaTypeDisposition(parts.shift()?.trim() ?? '')
+  if (mediatype === '') {
+    return ['', new Map(), ErrInvalidMediaParameter]
+  }
+  const params = new Map<string, string>()
+  for (let i = 0; i < parts.length; i++) {
+    const part = parts[i]
+    if (part.trim() === '' && i === parts.length - 1) {
+      break
+    }
+    const eq = part.indexOf('=')
+    if (eq < 0) {
+      return [mediatype, params, ErrInvalidMediaParameter]
+    }
+    const key = part.slice(0, eq).trim().toLowerCase()
+    const rawValue = part.slice(eq + 1).trim()
+    const value = parseParamValue(rawValue)
+    if (!isToken(key)) {
+      return [mediatype, params, ErrInvalidMediaParameter]
+    }
+    if (value === null) {
+      return [mediatype, params, ErrInvalidMediaParameter]
+    }
+    if (key.endsWith('*')) {
+      const decoded = decode2231Value(value)
+      if (decoded !== null) {
+        if (params.has(key.slice(0, -1)) && params.get(key.slice(0, -1)) !== decoded) {
+          return ['', new Map(), ErrInvalidMediaParameter]
+        }
+        params.set(key.slice(0, -1), decoded)
+      }
+      continue
+    }
+    if (params.has(key) && params.get(key) !== value) {
+      return ['', new Map(), ErrInvalidMediaParameter]
+    }
+    params.set(key, value)
+  }
+  return [mediatype, params, null]
+}
+
+export function AddExtensionType(ext: string, typ: string): $.GoError {
+  if (!ext.startsWith('.')) {
+    return ErrInvalidMediaParameter
+  }
+  const [, , err] = ParseMediaType(typ)
+  if (err !== null) {
+    return err
+  }
+  extensionTypes.set(ext.toLowerCase(), typ)
+  return null
+}
+
+export function ExtensionsByType(typ: string): [$.Slice<string>, $.GoError] {
+  const [mediaType, , err] = ParseMediaType(typ)
+  if (err !== null) {
+    return [null, err]
+  }
+  const out: string[] = []
+  for (const [ext, extType] of extensionTypes) {
+    const [known] = ParseMediaType(extType)
+    if (known === mediaType) {
+      out.push(ext)
+    }
+  }
+  out.sort()
+  return [$.arrayToSlice(out), null]
+}
+
+export function TypeByExtension(ext: string): string {
+  return extensionTypes.get(ext.toLowerCase()) ?? ''
+}
+
+export class WordDecoder {
+  public CharsetReader:
+    | ((charset: string, input: io.Reader) => [io.Reader | null, $.GoError])
+    | null = null
+
+  public Decode(word: string): [string, $.GoError] {
+    const decoded = decodeWord(word)
+    if (decoded === null) {
+      return ['', ErrInvalidMediaParameter]
+    }
+    return [decoded, null]
+  }
+
+  public DecodeHeader(header: string): [string, $.GoError] {
+    return [
+      header.replace(/=\?[^?\s]+\?[bBqQ]\?[^?\s]*\?=/g, (word) => {
+        return decodeWord(word) ?? word
+      }),
+      null,
+    ]
+  }
+}
+
+export function WordEncoder_Encode(
+  e: WordEncoder,
+  charset: string,
+  s: string,
+): string {
+  if (s === '') {
+    return ''
+  }
+  if (!needsWordEncoding(s)) {
+    return s
+  }
+  const encoding = String.fromCharCode(e).toLowerCase() === 'q' ? 'Q' : 'B'
+  const body =
+    encoding === 'Q' ? qEncode(s) : base64Encode(new TextEncoder().encode(s))
+  return `=?${charset}?${encoding}?${body}?=`
+}
+
+function needsWordEncoding(value: string): boolean {
+  for (const char of value) {
+    const code = char.codePointAt(0) ?? 0
+    if ((code < 0x20 || code > 0x7e) && char !== '\t') {
+      return true
+    }
+  }
+  return false
+}
+
 function formatParam(key: string, value: string): string {
+  if (needsWordEncoding(value)) {
+    return `; ${key}*=utf-8''${percentEncode(value)}`
+  }
   if (isToken(value)) {
     return `; ${key}=${value}`
   }
-  return `; ${key}*=utf-8''${percentEncode(value)}`
+  return `; ${key}="${value.replace(/(["\\])/g, '\\$1')}"`
 }
 
 function isToken(value: string): boolean {
-  return /^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$/.test(value)
+  return /^[!#$%&'*+\-.^_`{}|~0-9A-Za-z]+$/.test(value)
+}
+
+function parseParamValue(value: string): string | null {
+  if (value.startsWith('"')) {
+    return parseQuotedParamValue(value)
+  }
+  return isToken(value) ? value : null
+}
+
+function parseQuotedParamValue(value: string): string | null {
+  let out = ''
+  for (let i = 1; i < value.length; i++) {
+    const char = value[i]
+    if (char === '"') {
+      return i === value.length - 1 ? out : null
+    }
+    if (char === '\\' && i + 1 < value.length && isTSpecial(value[i + 1])) {
+      out += value[i + 1]
+      i++
+      continue
+    }
+    if (char === '\r' || char === '\n') {
+      return null
+    }
+    out += char
+  }
+  return null
+}
+
+function isTSpecial(value: string): boolean {
+  return '()<>@,;:\\"/[]?='.includes(value)
+}
+
+function formatMediaTypeDisposition(value: string): string {
+  const slash = value.indexOf('/')
+  if (slash < 0) {
+    return isToken(value) ? value.toLowerCase() : ''
+  }
+  if (slash === 0 || slash === value.length - 1) {
+    return ''
+  }
+  const major = value.slice(0, slash)
+  const sub = value.slice(slash + 1)
+  if (!isToken(major) || !isToken(sub)) {
+    return ''
+  }
+  return `${major.toLowerCase()}/${sub.toLowerCase()}`
 }
 
 function percentEncode(value: string): string {
@@ -58,3 +277,147 @@ function percentEncode(value: string): string {
   }
   return out
 }
+
+function decode2231Value(value: string): string | null {
+  const first = value.indexOf("'")
+  if (first < 0) {
+    return null
+  }
+  const second = value.indexOf("'", first + 1)
+  if (second < 0) {
+    return null
+  }
+  const charset = value.slice(0, first).toLowerCase()
+  if (charset !== 'utf-8' && charset !== 'us-ascii') {
+    return null
+  }
+  const bytes = percentDecode(value.slice(second + 1))
+  if (bytes === null) {
+    return null
+  }
+  if (charset === 'us-ascii' && bytes.some((byte) => byte > 0x7f)) {
+    return null
+  }
+  return new TextDecoder('utf-8').decode(bytes)
+}
+
+function percentDecode(value: string): Uint8Array | null {
+  const bytes: number[] = []
+  for (let i = 0; i < value.length; i++) {
+    if (value[i] !== '%') {
+      bytes.push(value.charCodeAt(i))
+      continue
+    }
+    if (i + 2 >= value.length || !isHex(value[i + 1]) || !isHex(value[i + 2])) {
+      return null
+    }
+    bytes.push(Number.parseInt(value.slice(i + 1, i + 3), 16))
+    i += 2
+  }
+  return new Uint8Array(bytes)
+}
+
+function isHex(char: string): boolean {
+  return /^[0-9A-Fa-f]$/.test(char)
+}
+
+function splitMediaType(value: string): string[] {
+  const parts: string[] = []
+  let current = ''
+  let quoted = false
+  for (let i = 0; i < value.length; i++) {
+    const char = value[i]
+    if (char === '"' && value[i - 1] !== '\\') {
+      quoted = !quoted
+    }
+    if (char === ';' && !quoted) {
+      parts.push(current)
+      current = ''
+      continue
+    }
+    current += char
+  }
+  parts.push(current)
+  return parts
+}
+
+function base64Encode(bytes: Uint8Array): string {
+  const chars = Array.from(bytes, (byte) => String.fromCharCode(byte)).join('')
+  if (typeof btoa === 'function') {
+    return btoa(chars)
+  }
+  const buffer = nodeBuffer()
+  if (buffer != null) {
+    return buffer.from(bytes).toString('base64')
+  }
+  throw new Error('mime: base64 encoder unavailable')
+}
+
+function base64Decode(text: string): Uint8Array {
+  if (typeof atob === 'function') {
+    return new Uint8Array(Array.from(atob(text), (char) => char.charCodeAt(0)))
+  }
+  const buffer = nodeBuffer()
+  if (buffer != null) {
+    return new Uint8Array(buffer.from(text, 'base64'))
+  }
+  throw new Error('mime: base64 decoder unavailable')
+}
+
+function nodeBuffer(): nodeBufferConstructor | null {
+  const processObj = (globalThis as { process?: nodeProcess }).process
+  return processObj?.getBuiltinModule?.('buffer')?.Buffer ?? null
+}
+
+function qEncode(value: string): string {
+  let out = ''
+  for (const byte of new TextEncoder().encode(value)) {
+    if (byte === 0x20) {
+      out += '_'
+      continue
+    }
+    if (
+      (byte >= 0x30 && byte <= 0x39) ||
+      (byte >= 0x41 && byte <= 0x5a) ||
+      (byte >= 0x61 && byte <= 0x7a)
+    ) {
+      out += String.fromCharCode(byte)
+      continue
+    }
+    out += `=${byte.toString(16).toUpperCase().padStart(2, '0')}`
+  }
+  return out
+}
+
+function qDecode(value: string): string {
+  const bytes: number[] = []
+  for (let i = 0; i < value.length; i++) {
+    const char = value[i]
+    if (char === '_') {
+      bytes.push(0x20)
+      continue
+    }
+    if (char === '=' && i + 2 < value.length) {
+      bytes.push(Number.parseInt(value.slice(i + 1, i + 3), 16))
+      i += 2
+      continue
+    }
+    bytes.push(char.charCodeAt(0))
+  }
+  return new TextDecoder().decode(new Uint8Array(bytes))
+}
+
+function decodeWord(word: string): string | null {
+  const match = /^=\?([^?]+)\?([bBqQ])\?([^?]*)\?=$/.exec(word)
+  if (match === null) {
+    return null
+  }
+  const [, charset, encoding, body] = match
+  if (!/^utf-?8$|^us-ascii$/i.test(charset)) {
+    return null
+  }
+  if (encoding.toLowerCase() === 'b') {
+    return new TextDecoder().decode(base64Decode(body))
+  }
+  return qDecode(body)
+}

package/gs/mime/parity.json

@@ -0,0 +1,36 @@
+{
+  "schemaVersion": 1,
+  "strict": true,
+  "symbols": {
+    "AddExtensionType": {
+      "status": "real"
+    },
+    "BEncoding": {
+      "status": "real"
+    },
+    "ErrInvalidMediaParameter": {
+      "status": "real"
+    },
+    "ExtensionsByType": {
+      "status": "real"
+    },
+    "FormatMediaType": {
+      "status": "real"
+    },
+    "ParseMediaType": {
+      "status": "real"
+    },
+    "QEncoding": {
+      "status": "real"
+    },
+    "TypeByExtension": {
+      "status": "real"
+    },
+    "WordDecoder": {
+      "status": "real"
+    },
+    "WordEncoder": {
+      "status": "real"
+    }
+  }
+}

package/gs/net/http/httptest/index.test.ts

@@ -1,16 +1,30 @@
 import { describe, expect, it } from 'vitest'
 
 import * as $ from '@goscript/builtin/index.js'
+import * as bytes from '@goscript/bytes/index.js'
+import * as context from '@goscript/context/index.js'
+import * as io from '@goscript/io/index.js'
 
 import {
   Handler,
   Header_Get,
   Header_Set,
   MethodGet,
+  MethodPost,
+  NoBody,
   NewRequest,
   StatusPartialContent,
 } from '../index.js'
-import { NewServer, NewUnstartedServer, Server_Start } from './index.js'
+import {
+  DefaultRemoteAddr,
+  NewRequest as NewTestRequest,
+  NewRecorder,
+  NewRequestWithContext,
+  NewServer,
+  NewTLSServer,
+  NewUnstartedServer,
+  Server_Start,
+} from './index.js'
 
 describe('net/http/httptest override', () => {
   it('exports server helpers for typechecked server tests', () => {
@@ -19,20 +33,59 @@ describe('net/http/httptest override', () => {
     }
 
     const srv = NewServer(handler)
+    const tlsSrv = NewTLSServer(handler)
     expect(srv.URL).toMatch(/^http:\/\/goscript-httptest-\d+\.invalid$/)
+    expect(tlsSrv.URL).toMatch(/^https:\/\/goscript-httptest-\d+\.invalid$/)
     expect(srv.Client()).toBeTruthy()
     expect(srv.Config().Handler).toBe(handler)
+    expect(tlsSrv.Client()).toBeTruthy()
     expect(Server_Start(NewUnstartedServer(handler))?.Error()).toBe(
       'net/http/httptest: Server.Start is not implemented in GoScript',
     )
     srv.Close()
+    tlsSrv.Close()
+  })
+
+  it('exports request defaults and recorder compatibility helpers', () => {
+    const req = NewRequestWithContext(context.Background(), '', 'https://example.invalid/path', null)
+    expect(DefaultRemoteAddr).toBe('1.2.3.4')
+    expect(req.Method).toBe(MethodGet)
+    expect(req.Host).toBe('example.invalid')
+    expect(req.RemoteAddr).toBe('192.0.2.1:1234')
+    expect(req.TLS).not.toBeNull()
+
+    const pathReq = NewRequestWithContext(context.Background(), MethodGet, '/local?x=1', null)
+    expect(pathReq.Host).toBe('example.com')
+    expect(pathReq.RequestURI).toBe('/local?x=1')
+    expect(pathReq.URL.Host).toBe('')
+    expect(pathReq.URL.Scheme).toBe('')
+
+    const bodyReq = NewTestRequest(MethodPost, '/body', bytes.NewReader($.stringToBytes('abc')))
+    expect(bodyReq.ContentLength).toBe(3)
+    const noBodyReq = NewTestRequest(MethodPost, '/empty', NoBody)
+    expect(noBodyReq.ContentLength).toBe(0)
+    const unknownReq = NewTestRequest(MethodPost, '/unknown', {
+      Read: () => [0, io.EOF],
+    })
+    expect(unknownReq.ContentLength).toBe(-1)
+
+    const recorder = NewRecorder()
+    recorder.WriteString('ok')
+    recorder.Flush()
+    expect(recorder.Code).toBe(200)
+    expect(recorder.Flushed).toBe(true)
+    expect(Buffer.from(recorder.Body.Bytes()).toString('utf8')).toBe('ok')
   })
 
   it('routes Client.Do through the in-memory server handler', async () => {
     const srv = NewServer({
       ServeHTTP(w, r) {
+        const req = $.pointerValue(r)!
+        expect(req.RequestURI).toBe('/pack.kvf')
+        expect(req.URL.Host).toBe('')
+        expect(req.Host).toMatch(/^goscript-httptest-\d+\.invalid$/)
         Header_Set(w!.Header(), 'Content-Range', 'bytes 0-3/4')
-        const range = Header_Get($.pointerValue(r)!.Header, 'Range')
+        const range = Header_Get(req.Header, 'Range')
         if (range !== 'bytes=0-3') {
           w!.WriteHeader(400)
           w!.Write($.stringToBytes(range))
@@ -82,4 +135,47 @@ describe('net/http/httptest override', () => {
     expect(Buffer.from(buf).toString('utf8')).toBe('async')
     srv.Close()
   })
+
+  it('closes request bodies through Server.Client transport', async () => {
+    const srv = NewServer({
+      ServeHTTP(w) {
+        w!.WriteHeader(StatusPartialContent)
+      },
+    })
+    let closed = false
+    const body = {
+      Read: (_p: Uint8Array): [number, $.GoError] => [0, io.EOF],
+      Close: (): $.GoError => {
+        closed = true
+        return null
+      },
+    }
+    const [req, reqErr] = NewRequest(MethodPost, srv.URL + '/close', body)
+    expect(reqErr).toBeNull()
+
+    const [resp, err] = await srv.Client().Do(req)
+
+    expect(err).toBeNull()
+    expect(resp?.StatusCode).toBe(StatusPartialContent)
+    expect(closed).toBe(true)
+    srv.Close()
+  })
+
+  it('suppresses bodies for Server.Client HEAD requests', async () => {
+    const srv = NewServer({
+      ServeHTTP(w) {
+        w!.WriteHeader(StatusPartialContent)
+        w!.Write($.stringToBytes('hidden'))
+      },
+    })
+
+    const [resp, err] = await srv.Client().Head(srv.URL + '/head')
+
+    expect(err).toBeNull()
+    expect(resp?.StatusCode).toBe(StatusPartialContent)
+    const [n, readErr] = resp!.Body!.Read(new Uint8Array(8))
+    expect(n).toBe(0)
+    expect(readErr).toBe(io.EOF)
+    srv.Close()
+  })
 })