@subsquid/openreader 1.0.2 → 2.1.0

package/src/ir/connection.ts

@@ -0,0 +1,48 @@
+import {OrderBy, Where} from "./args"
+import {FieldRequest} from "./fields"
+
+
+export interface RelayConnectionRequest {
+    orderBy: OrderBy
+    where?: Where
+    first?: number
+    after?: string
+    edgeNode?: FieldRequest[]
+    edgeCursor?: boolean
+    pageInfo?: boolean
+    totalCount?: boolean
+}
+
+
+export interface RelayConnectionResponse {
+    edges?: RelayConnectionEdge[]
+    pageInfo?: Partial<RelayConnectionPageInfo>
+    totalCount?: number
+}
+
+
+export interface RelayConnectionEdge {
+    node?: unknown
+    cursor?: string
+}
+
+
+export interface RelayConnectionPageInfo {
+    hasNextPage: boolean
+    hasPreviousPage: boolean
+    startCursor: string
+    endCursor: string
+}
+
+
+export function decodeRelayConnectionCursor(cursor: string): number | undefined {
+    if (!/^\d+$/.test(cursor)) return undefined
+    let val = parseInt(cursor, 10)
+    if (Number.isSafeInteger(val) && val > 0) return val
+    return undefined
+}
+
+
+export function encodeRelayConnectionCursor(val: number): string {
+    return '' + val
+}

package/src/ir/fields.ts

@@ -0,0 +1,40 @@
+import {
+    EnumPropType,
+    FkPropType,
+    ListLookupPropType,
+    ListPropType,
+    LookupPropType,
+    ObjectPropType, Prop,
+    PropType,
+    ScalarPropType,
+    UnionPropType
+} from "../model"
+import {EntityListArguments} from "./args"
+
+
+export type FieldRequest = EntityListRequest | ObjectRequest | OpaqueRequest
+
+
+type Base<T> = T extends PropType ? {
+    field: string
+    kind: T['kind']
+    type: T
+    prop: Prop
+    aliases: string[]
+    ifType?: string
+    index: number
+} : never
+
+
+export type EntityListRequest = Base<ListLookupPropType> & {
+    children: FieldRequest[]
+    args?: EntityListArguments
+}
+
+
+export type ObjectRequest = Base<FkPropType | LookupPropType | ObjectPropType | UnionPropType> & {
+    children: FieldRequest[]
+}
+
+
+export type OpaqueRequest = Base<ScalarPropType | EnumPropType | ListPropType>

package/src/limit.size.ts

@@ -0,0 +1,130 @@
+import {unexpectedCase} from '@subsquid/util-internal'
+import {Where} from './ir/args'
+import {RelayConnectionRequest} from './ir/connection'
+import {FieldRequest} from './ir/fields'
+import {Model} from './model'
+import {getEntity} from './model.tools'
+
+
+export function getSize(model: Model, fields: FieldRequest[]): number {
+    let total = 0
+    for (let req of fields) {
+        let size = getFieldSize(model, req)
+        if (Number.isFinite(size)) {
+            total += size * req.aliases.length
+        } else {
+            return Infinity
+        }
+    }
+    return total
+}
+
+
+function getFieldSize(model: Model, req: FieldRequest): number {
+    switch(req.kind) {
+        case "scalar":
+        case "list":
+            return req.prop.byteWeight || 1
+        case "enum":
+            return 1
+        case "object":
+        case "fk":
+        case "lookup":
+        case "union":
+            return getSize(model, req.children) + 1
+        case "list-lookup":
+            return getEntityListSize(
+                model,
+                req.type.entity,
+                req.children,
+                Math.min(req.args?.limit ?? Infinity, req.prop.cardinality ?? Infinity),
+                req.args?.where
+            ) + 1
+        default:
+            throw unexpectedCase()
+    }
+}
+
+
+export function getEntityListSize(
+    model: Model,
+    entityName: string,
+    fields: FieldRequest[],
+    limit?: number,
+    where?: Where
+): number {
+    let cardinality = Math.min(
+        getEntityCardinality(model, entityName),
+        limit ?? Infinity,
+        getWhereCardinality(where)
+    )
+    if (Number.isFinite(cardinality)) {
+        return cardinality * Math.max(getSize(model, fields), 1)
+    } else {
+        return Infinity
+    }
+}
+
+
+function getWhereCardinality(where?: Where): number {
+    if (where == null) return Infinity
+    switch(where.op) {
+        case 'AND': {
+            let min = Infinity
+            for (let co of where.args) {
+                min = Math.min(min, getWhereCardinality(co))
+            }
+            return min
+        }
+        case 'OR': {
+            if (where.args.length == 0) return Infinity
+            let max = 0
+            for (let co of where.args) {
+                max = Math.max(max, getWhereCardinality(co))
+            }
+            return max
+        }
+        case 'eq':
+            if (where.field == 'id') {
+                return 1
+            } else {
+                return Infinity
+            }
+        case 'in':
+            if (where.field == 'id') {
+                return where.values.length
+            } else {
+                return Infinity
+            }
+        default:
+            return Infinity
+    }
+}
+
+
+export function getRelaySize(model: Model, entityName: string, req: RelayConnectionRequest): number {
+    let total = 0
+    let limit = Math.min(
+        getEntityCardinality(model, entityName),
+        req.first ?? 100,
+        getWhereCardinality(req.where)
+    )
+    if (req.edgeNode) {
+        total += limit * Math.max(getSize(model, req.edgeNode), 1)
+    }
+    if (req.edgeCursor) {
+        total += limit
+    }
+    if (req.pageInfo) {
+        total += 4
+    }
+    if (req.totalCount) {
+        total += 1
+    }
+    return total
+}
+
+
+function getEntityCardinality(model: Model, entityName: string): number {
+    return getEntity(model, entityName).cardinality ?? Infinity
+}

package/src/main.ts

@@ -1,53 +1,85 @@
-import {Pool} from "pg"
-import {createPoolConfig} from "./db"
-import {Dialect} from "./dialect"
-import {serve} from "./server"
-import {loadModel} from "./tools"
+import {createLogger} from '@subsquid/logger'
+import {runProgram} from '@subsquid/util-internal'
+import {nat, Url} from '@subsquid/util-internal-commander'
+import {waitForInterruption} from '@subsquid/util-internal-http-server'
+import {Command, Option} from 'commander'
+import {Pool} from 'pg'
+import {Dialect} from './dialect'
+import {serve} from './server'
+import {loadModel} from './tools'
 
 
-module.exports = function main() {
-    let args = process.argv.slice(2)
+const LOG = createLogger('sqd:openreader')
 
-    if (args.indexOf('--help') >= 0) {
-        help()
-        process.exit(1)
-    }
 
-    if (args.length != 1) {
-        help()
-        process.exit(1)
-    }
+runProgram(async () => {
+    let program = new Command()
 
-    let model = loadModel(args[0])
-    let db = new Pool(createPoolConfig())
-    let port = process.env.GRAPHQL_SERVER_PORT || 3000
-    let dialect: Dialect = process.env.DB_TYPE == 'cockroach' ? 'cockroach' : 'postgres'
-
-    serve({model, db, port, dialect}).then(
-        () => {
-            console.log('OpenReader is listening on port ' + port)
-        },
-        err => {
-            console.error(err)
-            process.exit(1)
-        }
+    program.description(`
+GraphQL server for postgres-compatible databases
+    `.trim())
+
+    program.requiredOption('-s, --schema <file>', 'a path to a file or folder with database description')
+    program.requiredOption('-d, --db-url <url>', 'database connection string', Url(['postgres:']))
+    program.addOption(
+        new Option('-t, --db-type <type>', 'database type').choices(['postgres', 'cockroach']).default('postgres')
     )
-}
+    program.option('-p, --port <number>', 'port to listen on', nat, 3000)
+    program.option('--max-request-size <kb>', 'max request size in kilobytes', nat, 256)
+    program.option('--max-root-fields <count>', 'max number of root fields in a query', nat)
+    program.option('--max-response-size <nodes>', 'max response size measured in nodes', nat)
+    program.option('--sql-statement-timeout <ms>', 'sql statement timeout in ms', nat)
+    program.option('--subscriptions', 'enable gql subscriptions')
+    program.option('--subscription-poll-interval <ms>', 'subscription poll interval in ms', nat, 1000)
+    program.option('--subscription-sql-statement-timeout <ms>', 'sql statement timeout for polling queries', nat)
+    program.option('--subscription-max-response-size <nodes>', 'max response size measured in nodes', nat)
 
+    let opts = program.parse().opts() as {
+        schema: string
+        dbUrl: string
+        dbType: Dialect
+        port: number
+        maxRequestSize: number
+        maxRootFields?: number
+        maxResponseSize?: number
+        sqlStatementTimeout?: number
+        subscriptions?: boolean
+        subscriptionPollInterval: number
+        subscriptionSqlStatementTimeout?: number
+        subscriptionMaxResponseSize?: number
+    }
 
-function help() {
-    console.error(`
-Usage:  openreader SCHEMA
+    let model = loadModel(opts.schema)
+
+    let connection = new Pool({
+        connectionString: opts.dbUrl,
+        statement_timeout: opts.sqlStatementTimeout || undefined
+    })
+
+    let subscriptionConnection: Pool | undefined
+    if (opts.subscriptions) {
+        subscriptionConnection = new Pool({
+            connectionString: opts.dbUrl,
+            statement_timeout: opts.subscriptionSqlStatementTimeout || opts.sqlStatementTimeout || undefined
+        })
+    }
 
-OpenCRUD GraphQL server.
+    let server = await serve({
+        model,
+        dialect: opts.dbType,
+        connection,
+        port: opts.port,
+        log: LOG,
+        maxRequestSizeBytes: opts.maxRequestSize * 1024,
+        maxRootFields: opts.maxRootFields,
+        maxResponseNodes: opts.maxResponseSize,
+        subscriptions: opts.subscriptions,
+        subscriptionPollInterval: opts.subscriptionPollInterval,
+        subscriptionConnection,
+        subscriptionMaxResponseNodes: opts.subscriptionMaxResponseSize
+    })
 
-Can be configured using the following environment variables:
+    LOG.info(`listening on port ${server.port}`)
 
-    DB_NAME
-    DB_USER
-    DB_PASS
-    DB_HOST
-    DB_PORT
-    GRAPHQL_SERVER_PORT
-`)
-}
+    return waitForInterruption(server)
+}, err => LOG.fatal(err))

package/src/{gql/schema.ts → model.schema.ts}

@@ -18,8 +18,8 @@ import {
     parse,
     validateSchema
 } from "graphql"
-import {Index, Model, Prop, PropType} from "../model"
-import {validateModel} from "../model.tools"
+import {Index, Model, Prop, PropType, Scalar} from "./model"
+import {validateModel} from "./model.tools"
 import {customScalars} from "./scalars"
 
 
@@ -29,6 +29,8 @@ const baseSchema = buildASTSchema(parse(`
     directive @unique on FIELD_DEFINITION
     directive @index(fields: [String!] unique: Boolean) on OBJECT | FIELD_DEFINITION
     directive @fulltext(query: String!) on FIELD_DEFINITION
+    directive @cardinality(value: Int!) on OBJECT | FIELD_DEFINITION
+    directive @byteWeight(value: Float!) on FIELD_DEFINITION
     directive @variant on OBJECT # legacy
     directive @jsonField on OBJECT # legacy
     scalar ID
@@ -75,11 +77,12 @@ function addEntityOrJsonObjectOrInterface(model: Model, type: GraphQLObjectType
     let properties: Record<string, Prop> = {}
     let interfaces: string[] = []
     let indexes: Index[] = type instanceof GraphQLObjectType ? checkEntityIndexes(type) : []
+    let cardinality = checkEntityCardinality(type)
     let description = type.description || undefined
 
     switch(kind) {
         case 'entity':
-            model[type.name] = {kind, properties, description, interfaces, indexes}
+            model[type.name] = {kind, properties, description, interfaces, indexes, ...cardinality}
             break
         case 'object':
             model[type.name] = {kind, properties, description, interfaces}
@@ -120,6 +123,10 @@ function addEntityOrJsonObjectOrInterface(model: Model, type: GraphQLObjectType
         let derivedFrom = checkDerivedFrom(type, f)
         let index = checkFieldIndex(type, f)
         let unique = index?.unique || false
+        let limits = {
+            ...checkByteWeightDirective(type, f),
+            ...checkCardinalityLimitDirective(type, f)
+        }
 
         if (index) {
             indexes.push(index)
@@ -137,10 +144,11 @@ function addEntityOrJsonObjectOrInterface(model: Model, type: GraphQLObjectType
             properties[key] = {
                 type: wrapWithList(list.nulls, {
                     kind: 'scalar',
-                    name: fieldType.name
+                    name: fieldType.name as Scalar
                 }),
                 nullable,
-                description
+                description,
+                ...limits
             }
         } else if (fieldType instanceof GraphQLEnumType) {
             addEnum(model, fieldType)
@@ -150,7 +158,8 @@ function addEntityOrJsonObjectOrInterface(model: Model, type: GraphQLObjectType
                     name: fieldType.name
                 }),
                 nullable,
-                description
+                description,
+                ...limits
             }
         } else if (fieldType instanceof GraphQLUnionType) {
             addUnion(model, fieldType)
@@ -160,7 +169,8 @@ function addEntityOrJsonObjectOrInterface(model: Model, type: GraphQLObjectType
                     name: fieldType.name
                 }),
                 nullable,
-                description
+                description,
+                ...limits
             }
         } else if (fieldType instanceof GraphQLObjectType) {
             if (isEntityType(fieldType)) {
@@ -205,7 +215,8 @@ function addEntityOrJsonObjectOrInterface(model: Model, type: GraphQLObjectType
                                 field: derivedFrom.field
                             },
                             nullable: false,
-                            description
+                            description,
+                            ...limits
                         }
                         break
                     default:
@@ -219,7 +230,8 @@ function addEntityOrJsonObjectOrInterface(model: Model, type: GraphQLObjectType
                         name: fieldType.name
                     }),
                     nullable,
-                    description
+                    description,
+                    ...limits
                 }
             }
         } else {
@@ -456,6 +468,57 @@ function checkDerivedFrom(type: GraphQLNamedType, f: GraphQLField<any, any>): {f
 }
 
 
+function checkEntityCardinality(type: GraphQLObjectType | GraphQLInterfaceType): {cardinality?: number} {
+    let directives = type.astNode?.directives?.filter(d => d.name.value == 'cardinality') || []
+    if (directives.length > 0 && !isEntityType(type)) {
+        throw new SchemaError(`@cardinality directive can be only applied to entities, but were applied to ${type.name}`)
+    }
+    if (directives.length > 1) throw new SchemaError(
+        `Multiple @cardinality directives where applied to ${type.name}`
+    )
+    if (directives.length == 0) return {}
+    let arg = assertNotNull(directives[0].arguments?.find(arg => arg.name.value == 'value'))
+    assert(arg.value.kind == 'IntValue')
+    let cardinality = parseInt(arg.value.value, 10)
+    if (cardinality < 0) throw new SchemaError(
+        `Incorrect @cardinality where applied to ${type.name}. Cardinality value must be positive.`
+    )
+    return {cardinality}
+}
+
+
+function checkCardinalityLimitDirective(type: GraphQLNamedType, f: GraphQLField<any, any>): {cardinality?: number} {
+    let directives = f.astNode?.directives?.filter(d => d.name.value == 'cardinality') || []
+    if (directives.length > 1) throw new SchemaError(
+        `Multiple @cardinality directives where applied to ${type.name}.${f.name}`
+    )
+    if (directives.length == 0) return {}
+    let arg = assertNotNull(directives[0].arguments?.find(arg => arg.name.value == 'value'))
+    assert(arg.value.kind == 'IntValue')
+    let cardinality = parseInt(arg.value.value, 10)
+    if (cardinality < 0) throw new SchemaError(
+        `Incorrect @cardinality where applied to ${type.name}.${f.name}. Cardinality value must be positive.`
+    )
+    return {cardinality}
+}
+
+
+function checkByteWeightDirective(type: GraphQLNamedType, f: GraphQLField<any, any>): {byteWeight?: number} {
+    let directives = f.astNode?.directives?.filter(d => d.name.value == 'byteWeight') || []
+    if (directives.length > 1) throw new SchemaError(
+        `Multiple @byteWeight directives where applied to ${type.name}.${f.name}`
+    )
+    if (directives.length == 0) return {}
+    let arg = assertNotNull(directives[0].arguments?.find(arg => arg.name.value == 'value'))
+    assert(arg.value.kind == 'FloatValue')
+    let byteWeight = parseFloat(arg.value.value)
+    if (byteWeight < 0) throw new SchemaError(
+        `Incorrect @byteWeight where applied to ${type.name}.${f.name}. Byte weight value must be positive.`
+    )
+    return {byteWeight}
+}
+
+
 function unsupportedFieldTypeError(propName: string): Error {
     return new SchemaError(`Property ${propName} has unsupported type`)
 }

package/src/model.ts

@@ -9,6 +9,7 @@ export interface Entity extends TypeMeta {
     properties: Record<Name, Prop>
     interfaces?: Name[]
     indexes?: Index[]
+    cardinality?: number
 }
 
 
@@ -50,6 +51,14 @@ export interface Prop {
      * Whether the values in the column must be unique. Applicable only to entities.
      */
     unique?: boolean
+    /**
+     * Characteristic number of elements in a `list-lookup` or in a `list` of objects
+     */
+    cardinality?: number
+    /**
+     * Relative byte size of a scalar value or scalar list
+     */
+    byteWeight?: number
 }
 
 
@@ -66,10 +75,13 @@ export type PropType =
 
 export interface ScalarPropType {
     kind: 'scalar'
-    name: Name
+    name: Scalar
 }
 
 
+export type Scalar = 'ID' | 'String' | 'Int' | 'Float' | 'Boolean' | 'DateTime' | 'BigInt' | 'JSON' | 'Bytes'
+
+
 export interface EnumPropType {
     kind: 'enum'
     name: Name

package/src/{orderBy.ts → opencrud/orderBy.ts}

@@ -1,14 +1,7 @@
 import assert from "assert"
-import type {Model} from "./model"
-import {getUnionProps} from "./model.tools"
-
-
-export type SortOrder = 'ASC' | 'DESC'
-
-
-export interface OrderBy {
-    [field: string]: SortOrder | OrderBy
-}
+import type {Model} from "../model"
+import {getUnionProps} from "../model.tools"
+import {OrderBy} from "../ir/args"
 
 
 /**