json-rest-api 1.0.26 → 1.0.28

package/docs/API.md

@@ -2052,6 +2052,11 @@ const result = await api.resources.articles.query({
 // GET /articles?fields[articles]=title,summary&fields[users]=name,avatar
 ```
 
+An unknown field raises `RestApiFieldsetError` with the stable code
+`REST_API_FIELDSET_INVALID` and HTTP status `400`. The same error is preserved
+for primary resources and included relationships, so transports can classify it
+without parsing error messages.
+
 ### Including Related Resources
 
 Load related resources in a single request:

package/docs/GUIDE/GUIDE_2_7_Pagination_And_Ordering.md

@@ -13,7 +13,7 @@ await api.addResource('posts', {
     content: { type: 'string', required: true },
     view_count: { type: 'number', default: 0, indexed: true },
     published_at: { type: 'dateTime', indexed: true },
-    created_at: { type: 'dateTime', defaultTo: Date.now, indexed: true }
+    created_at: { type: 'dateTime', temporalPrecision: 3, defaultTo: () => new Date().toISOString(), indexed: true }
   },
   relationships: {
     comments: { 
@@ -54,7 +54,7 @@ await api.addResource('comments', {
   schema: {
     content: { type: 'string', required: true },
     author_name: { type: 'string', required: true, max: 100 },
-    created_at: { type: 'dateTime', defaultTo: Date.now, indexed: true },
+    created_at: { type: 'dateTime', temporalPrecision: 3, defaultTo: () => new Date().toISOString(), indexed: true },
     likes: { type: 'number', default: 0, indexed: true },
     post_id: { type: 'id', belongsTo: 'posts', as: 'post', required: true }
   },

package/docs/GUIDE/GUIDE_3_Field_Transformations.md

@@ -970,7 +970,7 @@ await api.addResource('posts', {
     title: { type: 'string', required: true },
     content: { type: 'string', required: true },
     author_id: { type: 'number', belongsTo: 'authors', as: 'author' },
-    published_at: { type: 'dateTime', default: 'now()' }
+    published_at: { type: 'dateTime', temporalPrecision: 3, defaultTo: () => new Date().toISOString() }
   },
   tableName: 'posts'
 });
@@ -1064,7 +1064,7 @@ await api.addResource('posts', {
       getter: (value) => value?.trim()
     },
     author_id: { type: 'number', belongsTo: 'authors', as: 'author' },
-    published_at: { type: 'dateTime', default: 'now()' }
+    published_at: { type: 'dateTime', temporalPrecision: 3, defaultTo: () => new Date().toISOString() }
   },
   tableName: 'posts'
 });

package/docs/GUIDE/GUIDE_Y_Appendices.md

@@ -6,7 +6,7 @@ This document explains how the JSON REST API handles date and time values throug
 
 ## Supported Date/Time Types
 
-The API supports three temporal data types in schemas:
+The API supports three string temporal types and two numeric epoch types in schemas:
 
 ### 1. `date`
 - **Format**: `YYYY-MM-DD`
@@ -16,8 +16,8 @@ The API supports three temporal data types in schemas:
 - **JSON Output**: ISO 8601 date string
 
 ### 2. `dateTime`
-- **Format**: `YYYY-MM-DD HH:MM:SS` (input) / ISO 8601 (output)
-- **Example Input**: `2024-01-15 14:30:00`
+- **Format**: RFC 3339 with seconds and an explicit `Z` or `±HH:MM` offset
+- **Example Input**: `2024-01-15T14:30:00Z`
 - **Example Output**: `2024-01-15T14:30:00.000Z`
 - **Usage**: Timestamps, created/updated times, or any date with time
 - **Database Storage**: DATETIME (MySQL) or TIMESTAMP (PostgreSQL)
@@ -30,6 +30,13 @@ The API supports three temporal data types in schemas:
 - **Database Storage**: TIME column type
 - **JSON Output**: ISO 8601 time string
 
+### 4. `epochMilliseconds` and `epochSeconds`
+- **Format**: An integer number or canonical base-10 integer string
+- **Example**: `1767323045000` (`epochMilliseconds`) or `1767323045` (`epochSeconds`)
+- **Usage**: Systems whose temporal contract is explicitly a Unix epoch value
+- **Database Storage**: BIGINT for table-backed resources
+- **JSON Output**: Integer number in the same unit declared by the schema type
+
 ## Schema Definition
 
 Define date/time fields in your schema like this:
@@ -37,12 +44,18 @@ Define date/time fields in your schema like this:
 ```javascript
 const articleSchema = {
   publishedDate: { type: 'date', required: true },
-  createdAt: { type: 'dateTime', defaultTo: 'now()' },
-  updatedAt: { type: 'dateTime', defaultTo: 'now()' },
+  createdAt: { type: 'dateTime', temporalPrecision: 3, defaultTo: () => new Date().toISOString() },
+  updatedAt: { type: 'dateTime', temporalPrecision: 3, defaultTo: () => new Date().toISOString() },
   dailyPostTime: { type: 'time', nullable: true }
 };
 ```
 
+`Date.prototype.toISOString()` always emits three fractional-second digits, so
+use it directly with unspecified precision or `temporalPrecision: 3` (or
+higher). A lower precision requires a producer that returns no more than the
+configured number of digits. Validation rejects excess input precision rather
+than silently truncating it.
+
 ## Input Validation
 
 ### On Write Operations (POST/PUT/PATCH)
@@ -64,21 +77,14 @@ The API validates and normalizes date/time inputs:
 ```
 
 **Accepted Input Formats:**
-- **date**: 
-  - `YYYY-MM-DD` (parsed at UTC midnight)
-  - ISO 8601 date strings
-  - Any JavaScript Date parseable string
-- **dateTime**: 
-  - ISO 8601 strings (`2024-01-15T14:30:00Z`) - recommended
-  - `YYYY-MM-DD HH:MM:SS` (assumed UTC)
-  - JavaScript Date parseable strings
-  - Unix timestamps (as numbers)
-- **time**: 
-  - `HH:MM:SS` or `HH:MM`
-  - Extracted from datetime strings
+- **date**: a real calendar date in `YYYY-MM-DD` form
+- **dateTime**: an RFC 3339 string with seconds and an explicit timezone, such as `2024-01-15T14:30:00Z` or `2024-01-15T22:30:00+08:00`
+- **time**: an offset-free `HH:MM[:SS[.fraction]]` string
+
+JavaScript `Date` objects, SQL datetime strings, Unix timestamps, locale text, and other values accepted by `Date.parse()` are not valid JSON temporal inputs. Use `epochMilliseconds` or `epochSeconds` schema fields for Unix timestamps.
 
 **Storage Format:**
-All date/time values are converted to JavaScript Date objects before storage, allowing the database driver to handle the appropriate formatting for each database system.
+Schema validation preserves valid JSON temporal strings exactly. The built-in storage adapters convert `date` and `dateTime` strings to database-native `Date` values at the database boundary, while `time` remains a string. A custom field-level `storage.serialize` function receives the validated string and owns any custom database conversion.
 
 ## Output Normalization
 
@@ -105,9 +111,15 @@ All date/time values are normalized when returned from the API:
 ### Key Normalization Behaviors:
 
 1. **Boolean Normalization**: Database values of `1`/`0` are converted to `true`/`false`
-2. **Date Objects**: All date/time values are returned as JavaScript Date objects internally, then serialized to ISO 8601 strings in JSON responses
+2. **JSON Temporal Values**: `date`, `dateTime`, and `time` attributes are returned as strings that match the same public schema contract accepted on input
 3. **UTC Assumption**: MySQL DATETIME values (which lack timezone info) are assumed to be UTC
 
+Normalization runs once after the database read and again at the final response
+boundary. The final pass covers getters, computed fields, and finishing hooks
+that produce native `Date` values. A malformed non-null temporal value is not
+reported as `null`; it fails with `REST_API_TEMPORAL_DATA_INVALID` and HTTP 500
+so stored-data or enrichment defects remain visible.
+
 ## Database-Specific Handling
 
 ### MySQL
@@ -164,7 +176,7 @@ If migrating from a system that stores dates differently:
 
 1. **Local Time Storage**: Convert all dates to UTC before importing
 2. **String Storage**: Ensure strings match expected formats
-3. **Numeric Timestamps**: Use `timestamp` type for Unix timestamps
+3. **Numeric Timestamps**: Use `epochMilliseconds` or `epochSeconds`, matching the producer's actual unit
 
 ### Database Configuration
 
@@ -206,23 +218,24 @@ For optimal date handling, configure your database connection:
 
 ### Issue 3: Time Values Need Date Context
 **Symptom**: Can't perform date arithmetic on time-only values  
-**Cause**: Time values lack date context  
-**Solution**: The API attaches times to epoch date (1970-01-01) in UTC
+**Cause**: Time values intentionally have no date or timezone context
+**Solution**: Keep recurring wall-clock values as `time`; use `dateTime` when an instant is required
 
 ## Technical Implementation Details
 
 The date/time handling is implemented in two key areas:
 
 1. **Input Validation** (`json-rest-schema`):
-   - Validates format on write operations
-   - Converts all date inputs to JavaScript Date objects
-   - Ensures date-only values parse at UTC midnight
-   - Returns Date objects for storage (Knex handles DB-specific formatting)
+   - Validates strict JSON temporal strings on write operations
+   - Preserves accepted strings exactly
+   - Rejects JavaScript `Date` objects and permissive legacy date formats
 
-2. **Output Normalization** (`database-value-normalizers.js`):
+2. **Storage and Output Normalization** (`storage` adapters and `database-value-normalizers.js`):
+   - Converts validated `date` and `dateTime` strings at the database boundary
+   - Uses the same database-native representation for writes and query comparisons
    - Handles database-specific quirks (MySQL timezone issues)
-   - Ensures Date objects are properly created from database values
+   - Parses database date values before emitting schema-valid JSON strings
    - Fixes MySQL datetime strings by assuming UTC
    - Maintains consistency across different database engines
 
-This two-stage approach ensures data integrity on input and consistent formatting on output, regardless of the underlying database system. The key insight is that JavaScript Date objects are used as the common format throughout the pipeline, with database drivers handling the conversion to/from their native formats.
+This separation keeps the public JSON contract strict and serializable while still giving database drivers native values at the storage boundary.

package/index.js

@@ -33,6 +33,8 @@ export { S3Storage } from './plugins/storage/s3-storage.js'
 
 // Error classes for consumers who need them
 export {
+  RestApiFieldsetError,
+  RestApiTemporalDataError,
   RestApiValidationError,
   RestApiResourceError,
   RestApiPayloadError

package/lib/error-context.js

@@ -0,0 +1,21 @@
+import { HookedApiError } from 'hooked-api'
+
+export function isRestApiError (error) {
+  return error instanceof HookedApiError
+}
+
+/**
+ * Add operational context to unexpected failures without erasing typed API
+ * errors that callers and transports need to classify.
+ */
+export function wrapUnexpectedError (error, { message, context } = {}) {
+  if (isRestApiError(error)) {
+    return error
+  }
+
+  const wrappedError = new Error(`${message}: ${error.message}`, {
+    cause: error
+  })
+  wrappedError.context = context
+  return wrappedError
+}

package/lib/rest-api-errors.js

@@ -1,5 +1,8 @@
 import { HookedApiError } from 'hooked-api'
 
+export const REST_API_FIELDSET_ERROR_CODE = 'REST_API_FIELDSET_INVALID'
+export const REST_API_TEMPORAL_DATA_ERROR_CODE = 'REST_API_TEMPORAL_DATA_INVALID'
+
 /**
  * REST API specific error classes that can be mapped to HTTP status codes
  * by the HTTP plugin or other protocol plugins
@@ -90,3 +93,70 @@ export class RestApiPayloadError extends HookedApiError {
     }
   }
 }
+
+/**
+ * Invalid JSON:API sparse fieldset.
+ *
+ * The structured code lets transports distinguish invalid client input from
+ * query or include failures without parsing the human-readable message.
+ */
+export class RestApiFieldsetError extends HookedApiError {
+  constructor ({ field, resourceType } = {}) {
+    super(
+      `Unknown sparse field '${field}' requested for '${resourceType}'`,
+      REST_API_FIELDSET_ERROR_CODE
+    )
+    this.type = 'rest_api_fieldset'
+    this.statusCode = 400
+    this.details = {
+      field,
+      resourceType
+    }
+  }
+
+  toJSON () {
+    return {
+      name: this.name,
+      message: this.message,
+      code: this.code,
+      type: this.type,
+      statusCode: this.statusCode,
+      details: this.details,
+      stack: this.stack
+    }
+  }
+}
+
+/**
+ * A non-null temporal value produced by storage or response enrichment does
+ * not satisfy the resource's public temporal contract.
+ */
+export class RestApiTemporalDataError extends HookedApiError {
+  constructor ({ field, resourceType, fieldType, source = 'database' } = {}) {
+    const fieldPath = [resourceType, field].filter(Boolean).join('.') || 'unknown field'
+    super(
+      `Invalid ${source} value for ${fieldType || 'temporal'} field '${fieldPath}'.`,
+      REST_API_TEMPORAL_DATA_ERROR_CODE
+    )
+    this.type = 'rest_api_temporal_data'
+    this.statusCode = 500
+    this.details = {
+      field,
+      resourceType,
+      fieldType,
+      source
+    }
+  }
+
+  toJSON () {
+    return {
+      name: this.name,
+      message: this.message,
+      code: this.code,
+      type: this.type,
+      statusCode: this.statusCode,
+      details: this.details,
+      stack: this.stack
+    }
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "json-rest-api",
-  "version": "1.0.26",
+  "version": "1.0.28",
   "main": "index.js",
   "type": "module",
   "scripts": {
@@ -29,7 +29,7 @@
   "description": "REST API plugin for hooked-api with JSON:API compliance",
   "dependencies": {
     "hooked-api": "^1.0.24",
-    "json-rest-schema": "^1.0.14",
+    "json-rest-schema": "^1.0.17",
     "semver": "^7.7.2"
   },
   "peerDependencies": {

package/plugins/core/connectors/lib/transport-http-helpers.js

@@ -1,3 +1,8 @@
+import {
+  REST_API_FIELDSET_ERROR_CODE,
+  REST_API_TEMPORAL_DATA_ERROR_CODE
+} from '../../../../lib/rest-api-errors.js'
+
 const JSON_API_WRITE_CONTENT_TYPES = [
   'application/vnd.api+json',
   'application/json'
@@ -63,6 +68,22 @@ export function mapRestApiErrorToHttp (error) {
       detail: issue.message,
       ...(issue.instancePath ? { source: { pointer: issue.instancePath } } : {})
     }))
+  } else if (error.code === REST_API_FIELDSET_ERROR_CODE) {
+    status = 400
+    errors = [{
+      status: '400',
+      code: error.code,
+      title: 'Invalid Sparse Fieldset',
+      detail: error.message
+    }]
+  } else if (error.code === REST_API_TEMPORAL_DATA_ERROR_CODE) {
+    status = 500
+    errors = [{
+      status: '500',
+      code: error.code,
+      title: 'Invalid Temporal Data',
+      detail: error.message
+    }]
   } else if (error.code === 'REST_API_VALIDATION') {
     status = 422
     if (error.details?.violations?.length) {

package/plugins/core/lib/anyapi/schema-utils.js

@@ -219,8 +219,9 @@ export const TYPE_TO_POOL = new Map([
   ['date', 'date'],
   ['datetime', 'date'],
   ['dateTime', 'date'],
-  ['timestamp', 'date'],
   ['time', 'date'],
+  ['epochMilliseconds', 'number'],
+  ['epochSeconds', 'number'],
   ['json', 'json'],
   ['object', 'json'],
 ])

package/plugins/core/lib/anyapi/utils/descriptor-helpers.js

@@ -1,5 +1,5 @@
 import { RestApiValidationError } from '../../../../../lib/rest-api-errors.js'
-import { normalizeDateValue } from '../../querying-writing/database-value-normalizers.js'
+import { normalizeValueForDatabaseStorage } from '../../querying-writing/database-value-normalizers.js'
 
 export const normalizeId = (value) => (value === null || value === undefined ? null : String(value))
 
@@ -93,7 +93,9 @@ export const coerceValueForDefinition = (value, definition, { isRelationship } =
   }
 
   if (['date', 'dateTime', 'time'].includes(type)) {
-    return normalizeDateValue(value, type)
+    return normalizeValueForDatabaseStorage(value, type, {
+      temporalPrecision: definition?.temporalPrecision
+    })
   }
 
   return value

package/plugins/core/lib/dbTablesOperations.js

@@ -300,7 +300,8 @@ function normalizeColumnShape (definition = {}) {
       return 'datetime'
     case 'time':
       return 'time'
-    case 'timestamp':
+    case 'epochMilliseconds':
+    case 'epochSeconds':
       return 'integer'
     case 'array':
     case 'object':
@@ -344,7 +345,8 @@ function normalizeDesiredTypeKind (definition = {}) {
     case 'blob':
     case 'file':
       return 'binary'
-    case 'timestamp':
+    case 'epochMilliseconds':
+    case 'epochSeconds':
       return 'integer'
     case 'string':
     case 'none':
@@ -361,6 +363,10 @@ function normalizeComparableDefault (value) {
   return String(value)
 }
 
+const hasStaticDefault = (definition = {}) => (
+  definition.defaultTo !== undefined && typeof definition.defaultTo !== 'function'
+)
+
 function normalizeDesiredColumn (tableContext, fieldName, definition, options = {}) {
   const nullable = definition.nullable === true
     ? true
@@ -375,8 +381,10 @@ function normalizeDesiredColumn (tableContext, fieldName, definition, options =
     typeKind: normalizeDesiredTypeKind(definition),
     nullable,
     required: definition.required === true,
-    hasDefault: definition.defaultTo !== undefined,
-    defaultValue: normalizeComparableDefault(definition.defaultTo),
+    hasDefault: hasStaticDefault(definition),
+    defaultValue: hasStaticDefault(definition)
+      ? normalizeComparableDefault(definition.defaultTo)
+      : undefined,
     unsigned: definition.type === 'id' ? definition.unsigned !== false : definition.unsigned === true,
     maxLength: definition.maxLength ?? null,
     numericPrecision: definition.precision ?? null,
@@ -499,8 +507,9 @@ function mapTypeToKnex (table, columnName, definition, options = {}) {
         ? table.time(columnName, { precision: temporalPrecision })
         : table.time(columnName)
 
-    case 'timestamp':
-      return table.integer(columnName)
+    case 'epochMilliseconds':
+    case 'epochSeconds':
+      return table.bigInteger(columnName)
 
     case 'array':
     case 'object':
@@ -524,7 +533,7 @@ function applyColumnConstraints (column, definition) {
     column.notNullable()
   }
 
-  if (definition.defaultTo !== undefined) {
+  if (hasStaticDefault(definition)) {
     column.defaultTo(definition.defaultTo)
   }
 
@@ -631,8 +640,9 @@ function buildColumnBuilderCode (columnName, definition, options = {}) {
         }
         break
 
-      case 'timestamp':
-        line = `table.integer(${quoteJsString(columnName)})`
+      case 'epochMilliseconds':
+      case 'epochSeconds':
+        line = `table.bigInteger(${quoteJsString(columnName)})`
         break
 
       case 'array':
@@ -658,7 +668,7 @@ function buildColumnBuilderCode (columnName, definition, options = {}) {
     line += '.notNullable()'
   }
 
-  if (definition.defaultTo !== undefined) {
+  if (hasStaticDefault(definition)) {
     line += `.defaultTo(${formatCodeLiteral(definition.defaultTo)})`
   }
 

package/plugins/core/lib/querying/knex-pagination-helpers.js

@@ -1,5 +1,6 @@
 import { getFieldValue } from '../storage/storage-mapping.js'
 import { serializeJsonApiQuery } from '../querying-writing/connectors-query-parser.js'
+import { normalizeDateValue } from '../querying-writing/database-value-normalizers.js'
 
 const buildJsonApiLink = (baseUrl, queryParams, page) => {
   const queryString = serializeJsonApiQuery(queryParams, { page })
@@ -236,9 +237,18 @@ export const generatePaginationLinks = (urlPrefix, scopeName, queryParams, pagin
 export const createCursor = (record, sortFields = ['id'], { schemaInfo = null } = {}) => {
   const parts = []
   sortFields.forEach(field => {
-    const value = schemaInfo
+    let value = schemaInfo
       ? getFieldValue(record, schemaInfo, field)
       : record[field]
+    const definition = schemaInfo?.schemaStructure?.[field]
+    if (['date', 'dateTime', 'time'].includes(definition?.type) && value != null) {
+      value = normalizeDateValue(value, definition.type, {
+        temporalPrecision: definition.temporalPrecision,
+        fieldName: field,
+        resourceType: schemaInfo?.scopeName || schemaInfo?.tableName,
+        source: 'cursor'
+      })
+    }
     if (value !== undefined) {
       const stringValue = value instanceof Date ? value.toISOString() : String(value)
       parts.push(`${field}:${encodeURIComponent(stringValue)}`)

package/plugins/core/lib/querying/knex-process-includes.js

@@ -1,4 +1,5 @@
 import { buildIncludedResources } from './knex-relationship-includes.js'
+import { isRestApiError, wrapUnexpectedError } from '../../../../lib/error-context.js'
 
 /**
  * Processes the ?include= parameter to load related resources efficiently
@@ -143,6 +144,8 @@ export const processIncludes = async (scope, records, deps) => {
 
     return includeResult.included
   } catch (error) {
+    if (isRestApiError(error)) throw error
+
     const { log, context } = deps || {}
 
     if (log) {
@@ -157,10 +160,13 @@ export const processIncludes = async (scope, records, deps) => {
       console.error('[PROCESS-INCLUDES] Error processing includes:', error)
     }
 
-    const enhancedError = new Error(
-      `Failed to process includes${context?.scopeName ? ` for scope '${context.scopeName}'` : ''}: ${error.message}`
-    )
-    enhancedError.originalError = error
-    throw enhancedError
+    throw wrapUnexpectedError(error, {
+      message: `Failed to process includes${context?.scopeName ? ` for scope '${context.scopeName}'` : ''}`,
+      context: {
+        scopeName: context?.scopeName,
+        includeParam: context?.queryParams?.include,
+        recordCount: records?.length || 0
+      }
+    })
   }
 }