@subsquid/openreader 2.0.0 → 2.1.0

package/src/main.ts

@@ -1,12 +1,12 @@
-import {createLogger} from "@subsquid/logger"
-import {runProgram} from "@subsquid/util-internal"
-import {nat, Url} from "@subsquid/util-internal-commander"
-import {waitForInterruption} from "@subsquid/util-internal-http-server"
-import {Command, Option} from "commander"
-import {Pool} from "pg"
-import {Dialect} from "./dialect"
-import {serve} from "./server"
-import {loadModel} from "./tools"
+import {createLogger} from '@subsquid/logger'
+import {runProgram} from '@subsquid/util-internal'
+import {nat, Url} from '@subsquid/util-internal-commander'
+import {waitForInterruption} from '@subsquid/util-internal-http-server'
+import {Command, Option} from 'commander'
+import {Pool} from 'pg'
+import {Dialect} from './dialect'
+import {serve} from './server'
+import {loadModel} from './tools'
 
 
 const LOG = createLogger('sqd:openreader')
@@ -26,10 +26,13 @@ GraphQL server for postgres-compatible databases
     )
     program.option('-p, --port <number>', 'port to listen on', nat, 3000)
     program.option('--max-request-size <kb>', 'max request size in kilobytes', nat, 256)
+    program.option('--max-root-fields <count>', 'max number of root fields in a query', nat)
+    program.option('--max-response-size <nodes>', 'max response size measured in nodes', nat)
     program.option('--sql-statement-timeout <ms>', 'sql statement timeout in ms', nat)
     program.option('--subscriptions', 'enable gql subscriptions')
     program.option('--subscription-poll-interval <ms>', 'subscription poll interval in ms', nat, 1000)
     program.option('--subscription-sql-statement-timeout <ms>', 'sql statement timeout for polling queries', nat)
+    program.option('--subscription-max-response-size <nodes>', 'max response size measured in nodes', nat)
 
     let opts = program.parse().opts() as {
         schema: string
@@ -37,10 +40,13 @@ GraphQL server for postgres-compatible databases
         dbType: Dialect
         port: number
         maxRequestSize: number
+        maxRootFields?: number
+        maxResponseSize?: number
         sqlStatementTimeout?: number
         subscriptions?: boolean
         subscriptionPollInterval: number
         subscriptionSqlStatementTimeout?: number
+        subscriptionMaxResponseSize?: number
     }
 
     let model = loadModel(opts.schema)
@@ -65,9 +71,12 @@ GraphQL server for postgres-compatible databases
         port: opts.port,
         log: LOG,
         maxRequestSizeBytes: opts.maxRequestSize * 1024,
+        maxRootFields: opts.maxRootFields,
+        maxResponseNodes: opts.maxResponseSize,
         subscriptions: opts.subscriptions,
         subscriptionPollInterval: opts.subscriptionPollInterval,
-        subscriptionConnection
+        subscriptionConnection,
+        subscriptionMaxResponseNodes: opts.subscriptionMaxResponseSize
     })
 
     LOG.info(`listening on port ${server.port}`)

package/src/model.schema.ts

@@ -29,7 +29,7 @@ const baseSchema = buildASTSchema(parse(`
     directive @unique on FIELD_DEFINITION
     directive @index(fields: [String!] unique: Boolean) on OBJECT | FIELD_DEFINITION
     directive @fulltext(query: String!) on FIELD_DEFINITION
-    directive @cardinality(value: Int!) on FIELD_DEFINITION
+    directive @cardinality(value: Int!) on OBJECT | FIELD_DEFINITION
     directive @byteWeight(value: Float!) on FIELD_DEFINITION
     directive @variant on OBJECT # legacy
     directive @jsonField on OBJECT # legacy
@@ -77,11 +77,12 @@ function addEntityOrJsonObjectOrInterface(model: Model, type: GraphQLObjectType
     let properties: Record<string, Prop> = {}
     let interfaces: string[] = []
     let indexes: Index[] = type instanceof GraphQLObjectType ? checkEntityIndexes(type) : []
+    let cardinality = checkEntityCardinality(type)
     let description = type.description || undefined
 
     switch(kind) {
         case 'entity':
-            model[type.name] = {kind, properties, description, interfaces, indexes}
+            model[type.name] = {kind, properties, description, interfaces, indexes, ...cardinality}
             break
         case 'object':
             model[type.name] = {kind, properties, description, interfaces}
@@ -467,10 +468,29 @@ function checkDerivedFrom(type: GraphQLNamedType, f: GraphQLField<any, any>): {f
 }
 
 
+function checkEntityCardinality(type: GraphQLObjectType | GraphQLInterfaceType): {cardinality?: number} {
+    let directives = type.astNode?.directives?.filter(d => d.name.value == 'cardinality') || []
+    if (directives.length > 0 && !isEntityType(type)) {
+        throw new SchemaError(`@cardinality directive can be only applied to entities, but were applied to ${type.name}`)
+    }
+    if (directives.length > 1) throw new SchemaError(
+        `Multiple @cardinality directives where applied to ${type.name}`
+    )
+    if (directives.length == 0) return {}
+    let arg = assertNotNull(directives[0].arguments?.find(arg => arg.name.value == 'value'))
+    assert(arg.value.kind == 'IntValue')
+    let cardinality = parseInt(arg.value.value, 10)
+    if (cardinality < 0) throw new SchemaError(
+        `Incorrect @cardinality where applied to ${type.name}. Cardinality value must be positive.`
+    )
+    return {cardinality}
+}
+
+
 function checkCardinalityLimitDirective(type: GraphQLNamedType, f: GraphQLField<any, any>): {cardinality?: number} {
     let directives = f.astNode?.directives?.filter(d => d.name.value == 'cardinality') || []
     if (directives.length > 1) throw new SchemaError(
-        `Multiple @cardinality where applied to ${type.name}.${f.name}`
+        `Multiple @cardinality directives where applied to ${type.name}.${f.name}`
     )
     if (directives.length == 0) return {}
     let arg = assertNotNull(directives[0].arguments?.find(arg => arg.name.value == 'value'))
@@ -486,7 +506,7 @@ function checkCardinalityLimitDirective(type: GraphQLNamedType, f: GraphQLField<
 function checkByteWeightDirective(type: GraphQLNamedType, f: GraphQLField<any, any>): {byteWeight?: number} {
     let directives = f.astNode?.directives?.filter(d => d.name.value == 'byteWeight') || []
     if (directives.length > 1) throw new SchemaError(
-        `Multiple @byteWeight where applied to ${type.name}.${f.name}`
+        `Multiple @byteWeight directives where applied to ${type.name}.${f.name}`
     )
     if (directives.length == 0) return {}
     let arg = assertNotNull(directives[0].arguments?.find(arg => arg.name.value == 'value'))

package/src/model.ts

@@ -9,6 +9,7 @@ export interface Entity extends TypeMeta {
     properties: Record<Name, Prop>
     interfaces?: Name[]
     indexes?: Index[]
+    cardinality?: number
 }
 
 

package/src/opencrud/schema.ts

@@ -29,11 +29,13 @@ import {
 } from "graphql/type/definition"
 import {Context} from "../context"
 import {decodeRelayConnectionCursor, RelayConnectionRequest} from "../ir/connection"
+import {getEntityListSize, getRelaySize, getSize} from '../limit.size'
 import {Entity, Interface, JsonObject, Model, Prop} from "../model"
 import {getObject, getUnionProps} from "../model.tools"
 import {customScalars} from "../scalars"
 import {EntityByIdQuery, EntityConnectionQuery, EntityCountQuery, EntityListQuery, Query} from "../sql/query"
 import {Subscription} from "../subscription"
+import {Limit} from '../util/limit'
 import {getResolveTree, getTreeRequest, hasTreeRequest, simplifyResolveTree} from "../util/resolve-tree"
 import {ensureArray, identity} from "../util/util"
 import {getOrderByMapping, parseOrderBy} from "./orderBy"
@@ -406,13 +408,14 @@ export class SchemaBuilder {
     private installEntityList(entityName: string, query: GqlFieldMap, subscription: GqlFieldMap): void {
         let model = this.model
         let queryName = toPlural(toCamelCase(entityName))
-        let outputType = new GraphQLList(new GraphQLNonNull(this.get(entityName)))
+        let outputType = new GraphQLNonNull(new GraphQLList(new GraphQLNonNull(this.get(entityName))))
         let argsType = this.entityListArguments(entityName)
 
-        function createQuery(context: Context, info: GraphQLResolveInfo) {
+        function createQuery(context: Context, info: GraphQLResolveInfo, limit?: Limit) {
             let tree = getResolveTree(info)
             let fields = parseResolveTree(model, entityName, info.schema, tree)
             let args = parseEntityListArguments(model, entityName, tree.args)
+            limit?.check(() => getEntityListSize(model, entityName, fields, args.limit, args.where) + 1)
             return new EntityListQuery(
                 model,
                 context.openreader.dialect,
@@ -426,7 +429,7 @@ export class SchemaBuilder {
             type: outputType,
             args: argsType,
             resolve(source, args, context, info) {
-                let q = createQuery(context, info)
+                let q = createQuery(context, info, context.openreader.responseSizeLimit)
                 return context.openreader.executeQuery(q)
             }
         }
@@ -436,7 +439,7 @@ export class SchemaBuilder {
             args: argsType,
             resolve: identity,
             subscribe(source, args, context, info) {
-                let q = createQuery(context, info)
+                let q = createQuery(context, info, context.openreader.subscriptionResponseSizeLimit)
                 return context.openreader.subscription(q)
             }
         }
@@ -449,9 +452,10 @@ export class SchemaBuilder {
             id: {type: new GraphQLNonNull(GraphQLString)}
         }
 
-        function createQuery(context: Context, info: GraphQLResolveInfo) {
+        function createQuery(context: Context, info: GraphQLResolveInfo, limit?: Limit) {
             let tree = getResolveTree(info)
             let fields = parseResolveTree(model, entityName, info.schema, tree)
+            limit?.check(() => getSize(model, fields) + 1)
             return new EntityByIdQuery(
                 model,
                 context.openreader.dialect,
@@ -465,7 +469,7 @@ export class SchemaBuilder {
             type: this.get(entityName),
             args: argsType,
             async resolve(source, args, context, info) {
-                let q = createQuery(context, info)
+                let q = createQuery(context, info, context.openreader.responseSizeLimit)
                 return context.openreader.executeQuery(q)
             }
         }
@@ -475,7 +479,7 @@ export class SchemaBuilder {
             args: argsType,
             resolve: identity,
             subscribe(source, args, context, info) {
-                let q = createQuery(context, info)
+                let q = createQuery(context, info, context.openreader.subscriptionResponseSizeLimit)
                 return context.openreader.subscription(q)
             }
         }
@@ -493,6 +497,7 @@ export class SchemaBuilder {
             async resolve(source, args, context, info) {
                 let tree = getResolveTree(info)
                 let fields = parseResolveTree(model, entityName, info.schema, tree)
+                context.openreader.responseSizeLimit?.check(() => getSize(model, fields) + 1)
                 let query = new EntityListQuery(
                     model,
                     context.openreader.dialect,
@@ -577,6 +582,8 @@ export class SchemaBuilder {
                     }
                 }
 
+                context.openreader.responseSizeLimit?.check(() => getRelaySize(model, entityName, req) + 1)
+
                 let result = await context.openreader.executeQuery(new EntityConnectionQuery(
                     model,
                     context.openreader.dialect,

package/src/server.ts

@@ -1,21 +1,23 @@
-import type {Logger} from "@subsquid/logger"
-import {listen, ListeningServer} from "@subsquid/util-internal-http-server"
-import {PluginDefinition} from "apollo-server-core"
-import {ApolloServer} from "apollo-server-express"
-import express from "express"
-import fs from "fs"
-import {GraphQLSchema} from "graphql"
-import {useServer as useWsServer} from "graphql-ws/lib/use/ws"
-import http from "http"
-import path from "path"
-import type {Pool} from "pg"
-import {WebSocketServer} from "ws"
-import {Context} from "./context"
-import {PoolOpenreaderContext} from "./db"
-import type {Dialect} from "./dialect"
-import type {Model} from "./model"
-import {SchemaBuilder} from "./opencrud/schema"
-import {logGraphQLError} from "./util/error-handling"
+import type {Logger} from '@subsquid/logger'
+import {listen, ListeningServer} from '@subsquid/util-internal-http-server'
+import {PluginDefinition} from 'apollo-server-core'
+import {ApolloServer} from 'apollo-server-express'
+import express from 'express'
+import fs from 'fs'
+import {ExecutionArgs, GraphQLSchema} from 'graphql'
+import {useServer as useWsServer} from 'graphql-ws/lib/use/ws'
+import http from 'http'
+import path from 'path'
+import type {Pool} from 'pg'
+import {WebSocketServer} from 'ws'
+import {Context, OpenreaderContext} from './context'
+import {PoolOpenreaderContext} from './db'
+import type {Dialect} from './dialect'
+import type {Model} from './model'
+import {SchemaBuilder} from './opencrud/schema'
+import {logGraphQLError} from './util/error-handling'
+import {executeWithLimit} from './util/execute'
+import {ResponseSizeLimit} from './util/limit'
 
 
 export interface ServerOptions {
@@ -23,25 +25,46 @@ export interface ServerOptions {
     model: Model
     connection: Pool
     dialect?: Dialect
-    subscriptions?: boolean
-    subscriptionPollInterval?: number
-    subscriptionConnection?: Pool
     graphiqlConsole?: boolean
     log?: Logger
     maxRequestSizeBytes?: number
+    maxRootFields?: number
+    maxResponseNodes?: number
+    subscriptions?: boolean
+    subscriptionPollInterval?: number
+    subscriptionConnection?: Pool
+    subscriptionMaxResponseNodes?: number
 }
 
 
 export async function serve(options: ServerOptions): Promise<ListeningServer> {
-    let {connection, subscriptionConnection, subscriptionPollInterval} = options
+    let {connection, subscriptionConnection, subscriptionPollInterval, maxResponseNodes, subscriptionMaxResponseNodes} = options
     let dialect = options.dialect ?? 'postgres'
 
     let schema = new SchemaBuilder(options).build()
+
     let context = () => {
+        let openreader: OpenreaderContext = new PoolOpenreaderContext(
+            dialect,
+            connection,
+            subscriptionConnection,
+            subscriptionPollInterval
+        )
+
+        if (maxResponseNodes) {
+            openreader.responseSizeLimit = new ResponseSizeLimit(maxResponseNodes)
+            openreader.subscriptionResponseSizeLimit = new ResponseSizeLimit(maxResponseNodes)
+        }
+
+        if (subscriptionMaxResponseNodes) {
+            openreader.subscriptionResponseSizeLimit = new ResponseSizeLimit(subscriptionMaxResponseNodes)
+        }
+
         return {
-            openreader: new PoolOpenreaderContext(dialect, connection, subscriptionConnection, subscriptionPollInterval)
+            openreader
         }
     }
+
     let disposals: Dispose[] = []
 
     return addServerCleanup(disposals, runApollo({
@@ -52,7 +75,8 @@ export async function serve(options: ServerOptions): Promise<ListeningServer> {
         subscriptions: options.subscriptions,
         log: options.log,
         graphiqlConsole: options.graphiqlConsole,
-        maxRequestSizeBytes: options.maxRequestSizeBytes
+        maxRequestSizeBytes: options.maxRequestSizeBytes,
+        maxRootFields: options.maxRootFields
     }), options.log)
 }
 
@@ -70,15 +94,21 @@ export interface ApolloOptions {
     graphiqlConsole?: boolean
     log?: Logger
     maxRequestSizeBytes?: number
+    maxRootFields?: number
 }
 
 
 export async function runApollo(options: ApolloOptions): Promise<ListeningServer> {
-    let {disposals, context, schema, log} = options
+    const {disposals, context, schema, log, maxRootFields} = options
+
     let maxRequestSizeBytes = options.maxRequestSizeBytes ?? 256 * 1024
     let app = express()
     let server = http.createServer(app)
 
+    const execute = maxRootFields
+            ? (args: ExecutionArgs) => executeWithLimit(maxRootFields, args)
+            : undefined
+
     if (options.subscriptions) {
         let wsServer = new WebSocketServer({
             server,
@@ -89,6 +119,7 @@ export async function runApollo(options: ApolloOptions): Promise<ListeningServer
             {
                 schema,
                 context,
+                execute,
                 onError(ctx, message, errors) {
                     if (log) {
                         // FIXME: we don't want to log client errors
@@ -111,6 +142,16 @@ export async function runApollo(options: ApolloOptions): Promise<ListeningServer
         schema,
         context,
         stopOnTerminationSignals: false,
+        executor: execute && (async req => {
+            return execute({
+                schema,
+                document: req.document,
+                rootValue: {},
+                contextValue: req.context,
+                variableValues: req.request.variables,
+                operationName: req.operationName
+            })
+        }),
         plugins: [
             ...options.plugins || [],
             {
@@ -136,7 +177,6 @@ export async function runApollo(options: ApolloOptions): Promise<ListeningServer
         setupGraphiqlConsole(app)
     }
 
-
     await apollo.start()
     disposals.push(() => apollo.stop())
 

package/src/test/limits.test.ts

@@ -0,0 +1,163 @@
+import expect from 'expect'
+import {useDatabase, useServer} from './setup'
+
+
+describe('response size limits', function() {
+    useDatabase([
+        `create table "order1" (id text primary key)`,
+        `create table item1 (id text primary key, order_id text, name text)`,
+        `create table "order2" (id text primary key)`,
+        `create table item2 (id text primary key, order_id text, name text)`,
+        `create table "order3" (id text primary key)`,
+        `create table item3 (id text primary key, order_id text, name text)`,
+    ])
+
+    const client = useServer(`
+        type Order1 @entity {
+            id: ID!
+            items: [Item1!]! @derivedFrom(field: "order")
+        }
+        
+        type Item1 @entity {
+            id: ID!
+            order: Order1!
+            name: String
+        }
+        
+        type Order2 @entity @cardinality(value: 10) {
+            id: ID!
+            items: [Item2!]! @derivedFrom(field: "order")
+        }
+        
+        type Item2 @entity {
+            id: ID!
+            order: Order2!
+            name: String
+        }
+        
+        type Order3 @entity {
+            id: ID!
+            items: [Item3!]! @derivedFrom(field: "order") @cardinality(value: 10)
+        }
+        
+        type Item3 @entity {
+            id: ID!
+            order: Order3!
+            name: String @byteWeight(value: 10.0)
+        }
+    `, {
+        maxResponseNodes: 50,
+        maxRootFields: 3
+    })
+
+    it('unlimited requests fail', async function() {
+        let result = await client.query(`
+            query {
+                order1s {
+                    id
+                }
+            }
+        `)
+        expect(result).toMatchObject({
+            data: null,
+            errors: [
+                expect.objectContaining({message: 'response might exceed the size limit', path: ['order1s']})
+            ]
+        })
+    })
+
+    it('limited requests work', function() {
+        return client.test(`
+            query {
+                order1s(limit: 10) {
+                    items(limit: 2) {
+                        id
+                    }
+                }
+            }
+        `, {
+            order1s: []
+        })
+    })
+
+    it('entity level cardinalities are respected', function() {
+        return client.test(`
+            query {
+                order2s {
+                    id
+                }
+            }
+        `, {
+            order2s: []
+        })
+    })
+
+    it('item cardinalities are respected', function() {
+        return client.test(`
+            query {
+                order3s(limit: 1) {
+                    items { id }
+                }
+            }
+        `, {
+            order3s: []
+        })
+    })
+
+    it('@byteWeight annotations are respected', async function() {
+        let result = await client.query(`
+            query {
+                order3s(limit: 1) {
+                    items(limit: 8) { name }
+                }
+            }
+        `)
+        expect(result).toEqual({
+            data: null,
+            errors: [
+                expect.objectContaining({
+                    message: 'response might exceed the size limit',
+                    path: ['order3s']
+                })
+            ]
+        })
+        await client.test(`
+            query {
+                order3s(limit: 1) {
+                    items(limit: 4) { name }
+                }
+            }
+        `, {
+            order3s: []
+        })
+    })
+
+    it('id_in conditions are understood', function() {
+        return client.test(`
+            query {
+                order1s(where: {id_in: ["1", "2", "3"]}) {
+                    id
+                }
+            }
+        `, {
+            order1s: []
+        })
+    })
+
+    it('root query fields limit', async function() {
+        return client.errorTest(`
+            query {
+                a: order1ById(id: "1") { id }
+                b: order1ById(id: "1") { id }
+                c: order1ById(id: "1") { id }
+                d: order1ById(id: "1") { id }
+            }
+        `, {
+            errors: [
+                expect.objectContaining({
+                    message: 'only 3 root query fields allowed, but got 4'
+                })
+            ]
+        })
+    })
+})

package/src/test/setup.ts

@@ -4,7 +4,7 @@ import {Client} from "gql-test-client"
 import {parse} from "graphql"
 import {Client as PgClient, ClientBase, Pool} from "pg"
 import {buildModel, buildSchema} from "../model.schema"
-import {serve} from "../server"
+import {serve, ServerOptions} from '../server'
 
 
 export function isCockroach(): boolean {
@@ -59,7 +59,7 @@ export function useDatabase(sql: string[]): void {
 }
 
 
-export function useServer(schema: string): Client {
+export function useServer(schema: string, options?: Partial<ServerOptions>): Client {
     let client = new Client('not defined')
     let db = new Pool(db_config)
     let info: ListeningServer | undefined
@@ -69,7 +69,10 @@ export function useServer(schema: string): Client {
             model: buildModel(buildSchema(parse(schema))),
             port: 0,
             dialect: isCockroach() ? 'cockroach' : 'postgres',
-            subscriptions: true
+            subscriptions: true,
+            subscriptionPollInterval: 500,
+            maxRootFields: 10,
+            ...options
         })
         client.endpoint = `http://localhost:${info.port}/graphql`
     })

package/src/util/execute.ts

@@ -0,0 +1,53 @@
+import {getOperationRootType, GraphQLError} from 'graphql'
+import {ExecutionResult} from 'graphql-ws'
+import {
+    assertValidExecutionArguments,
+    buildExecutionContext,
+    collectFields,
+    execute as graphqlExecute,
+    ExecutionArgs,
+    ExecutionContext
+} from 'graphql/execution/execute'
+import {PromiseOrValue} from 'graphql/jsutils/PromiseOrValue'
+
+
+export function executeWithLimit(maxQueries: number, args: ExecutionArgs): PromiseOrValue<ExecutionResult> {
+    assertValidExecutionArguments(args.schema, args.document, args.variableValues)
+
+    let xtx = buildExecutionContext(
+        args.schema,
+        args.document,
+        args.rootValue,
+        args.contextValue,
+        args.variableValues,
+        args.operationName,
+        args.fieldResolver,
+        args.typeResolver
+    )
+
+    if (Array.isArray(xtx)) {
+        return {errors: xtx}
+    }
+
+    let etx = xtx as ExecutionContext
+    if (etx.operation.operation == 'query') {
+        let query = getOperationRootType(etx.schema, etx.operation)
+        let fields = collectFields(
+            etx,
+            query,
+            etx.operation.selectionSet,
+            Object.create(null),
+            Object.create(null)
+        )
+        let fieldsCount = Object.keys(fields).length
+        if (fieldsCount > maxQueries) {
+            return {
+                errors: [
+                    new GraphQLError(`only ${maxQueries} root query fields allowed, but got ${fieldsCount}`)
+                ]
+            }
+        }
+    }
+
+    return graphqlExecute(args)
+}

package/src/util/limit.ts

@@ -0,0 +1,34 @@
+import assert from 'assert'
+import {GraphQLError} from 'graphql'
+
+
+export class Limit {
+    constructor(private error: Error, private value: number) {
+        assert(this.value > 0)
+    }
+
+    get left(): number {
+        return Math.max(this.value, 0)
+    }
+
+    check(cb: (left: number) => number): void {
+        if (this.value < 0) throw this.error
+        let left = this.value - cb(this.value)
+        if (left < 0) {
+            throw this.error
+        } else {
+            this.value = left
+        }
+    }
+}
+
+
+const SIZE_LIMIT = new GraphQLError('response might exceed the size limit')
+SIZE_LIMIT.stack = undefined
+
+
+export class ResponseSizeLimit extends Limit {
+    constructor(maxNodes: number) {
+        super(SIZE_LIMIT, maxNodes)
+    }
+}