@subsquid/openreader 2.0.0 → 3.0.0

package/src/opencrud/tree.ts

@@ -1,29 +1,30 @@
-import {unexpectedCase} from "@subsquid/util-internal"
-import assert from "assert"
-import {GraphQLSchema} from "graphql"
-import {ResolveTree} from "graphql-parse-resolve-info"
-import {EntityListArguments} from "../ir/args"
-import {FieldRequest, OpaqueRequest} from "../ir/fields"
-import {Model} from "../model"
-import {simplifyResolveTree} from "../util/resolve-tree"
-import {ensureArray} from "../util/util"
-import {parseOrderBy} from "./orderBy"
-import {parseWhere} from "./where"
-
-
-export function parseResolveTree(
+import {unexpectedCase} from '@subsquid/util-internal'
+import assert from 'assert'
+import {GraphQLSchema} from 'graphql'
+import {ResolveTree} from 'graphql-parse-resolve-info'
+import {SqlArguments} from '../ir/args'
+import {AnyFields, FieldRequest, FieldsByEntity, OpaqueRequest} from '../ir/fields'
+import {Model} from '../model'
+import {getQueryableEntities} from '../model.tools'
+import {simplifyResolveTree} from '../util/resolve-tree'
+import {ensureArray} from '../util/util'
+import {parseOrderBy} from './orderBy'
+import {parseWhere} from './where'
+
+
+export function parseObjectTree(
     model: Model,
-    objectName: string,
+    typeName: string,
     schema: GraphQLSchema,
     tree: ResolveTree
 ): FieldRequest[] {
 
     let requests: FieldRequest[] = []
     let requestedScalars: Record<string, true> = {}
-    let object = model[objectName]
+    let object = model[typeName]
     assert(object.kind == "entity" || object.kind == "object")
 
-    let fields = simplifyResolveTree(schema, tree, objectName).fields
+    let fields = simplifyResolveTree(schema, tree, typeName).fields
     for (let alias in fields) {
         let f = fields[alias]
         let prop = object.properties[f.name]
@@ -51,7 +52,7 @@ export function parseResolveTree(
                     type: prop.type,
                     prop,
                     index: 0,
-                    children: parseResolveTree(model, prop.type.name, schema, f)
+                    children: parseObjectTree(model, prop.type.name, schema, f)
                 })
                 break
             case "union": {
@@ -59,7 +60,7 @@ export function parseResolveTree(
                 assert(union.kind == "union")
                 let children: FieldRequest[] = []
                 for (let variant of union.variants) {
-                    for (let req of parseResolveTree(model, variant, schema, f)) {
+                    for (let req of parseObjectTree(model, variant, schema, f)) {
                         req.ifType = variant
                         children.push(req)
                     }
@@ -83,7 +84,7 @@ export function parseResolveTree(
                     type: prop.type,
                     prop,
                     index: 0,
-                    children: parseResolveTree(model, prop.type.foreignEntity, schema, f)
+                    children: parseObjectTree(model, prop.type.entity, schema, f)
                 })
                 break
             case "lookup":
@@ -94,7 +95,7 @@ export function parseResolveTree(
                     type: prop.type,
                     prop,
                     index: 0,
-                    children: parseResolveTree(model, prop.type.entity, schema, f)
+                    children: parseObjectTree(model, prop.type.entity, schema, f)
                 })
                 break
             case "list-lookup":
@@ -105,8 +106,8 @@ export function parseResolveTree(
                     type: prop.type,
                     prop,
                     index: 0,
-                    args: parseEntityListArguments(model, prop.type.entity, f.args),
-                    children: parseResolveTree(model, prop.type.entity, schema, f)
+                    args: parseSqlArguments(model, prop.type.entity, f.args),
+                    children: parseObjectTree(model, prop.type.entity, schema, f)
                 })
                 break
             default:
@@ -118,8 +119,8 @@ export function parseResolveTree(
 }
 
 
-export function parseEntityListArguments(model: Model, entityName: string, gqlArgs: any): EntityListArguments {
-    let args: EntityListArguments = {}
+export function parseSqlArguments(model: Model, typeName: string, gqlArgs: any): SqlArguments {
+    let args: SqlArguments = {}
 
     let where = parseWhere(gqlArgs.where)
     if (where) {
@@ -127,7 +128,7 @@ export function parseEntityListArguments(model: Model, entityName: string, gqlAr
     }
 
     if (gqlArgs.orderBy) {
-        args.orderBy = parseOrderBy(model, entityName, ensureArray(gqlArgs.orderBy))
+        args.orderBy = parseOrderBy(model, typeName, ensureArray(gqlArgs.orderBy))
     }
 
     if (gqlArgs.offset) {
@@ -142,3 +143,31 @@ export function parseEntityListArguments(model: Model, entityName: string, gqlAr
 
     return args
 }
+
+
+export function parseQueryableTree(
+    model: Model,
+    queryableName: string,
+    schema: GraphQLSchema,
+    tree: ResolveTree
+): FieldsByEntity {
+    let fields: FieldsByEntity = {}
+    for (let entity of getQueryableEntities(model, queryableName)) {
+        fields[entity] = parseObjectTree(model, entity, schema, tree)
+    }
+    return fields
+}
+
+
+export function parseAnyTree(
+    model: Model,
+    typeName: string,
+    schema: GraphQLSchema,
+    tree: ResolveTree
+): AnyFields {
+    if (model[typeName].kind == 'interface') {
+        return parseQueryableTree(model, typeName, schema, tree)
+    } else {
+        return parseObjectTree(model, typeName, schema, tree)
+    }
+}

package/src/server.ts

@@ -1,21 +1,23 @@
-import type {Logger} from "@subsquid/logger"
-import {listen, ListeningServer} from "@subsquid/util-internal-http-server"
-import {PluginDefinition} from "apollo-server-core"
-import {ApolloServer} from "apollo-server-express"
-import express from "express"
-import fs from "fs"
-import {GraphQLSchema} from "graphql"
-import {useServer as useWsServer} from "graphql-ws/lib/use/ws"
-import http from "http"
-import path from "path"
-import type {Pool} from "pg"
-import {WebSocketServer} from "ws"
-import {Context} from "./context"
-import {PoolOpenreaderContext} from "./db"
-import type {Dialect} from "./dialect"
-import type {Model} from "./model"
-import {SchemaBuilder} from "./opencrud/schema"
-import {logGraphQLError} from "./util/error-handling"
+import type {Logger} from '@subsquid/logger'
+import {listen, ListeningServer} from '@subsquid/util-internal-http-server'
+import {PluginDefinition} from 'apollo-server-core'
+import {ApolloServer} from 'apollo-server-express'
+import express from 'express'
+import fs from 'fs'
+import {ExecutionArgs, GraphQLSchema} from 'graphql'
+import {useServer as useWsServer} from 'graphql-ws/lib/use/ws'
+import http from 'http'
+import path from 'path'
+import type {Pool} from 'pg'
+import {WebSocketServer} from 'ws'
+import {Context, OpenreaderContext} from './context'
+import {PoolOpenreaderContext} from './db'
+import type {Dialect} from './dialect'
+import type {Model} from './model'
+import {SchemaBuilder} from './opencrud/schema'
+import {logGraphQLError} from './util/error-handling'
+import {executeWithLimit} from './util/execute'
+import {ResponseSizeLimit} from './util/limit'
 
 
 export interface ServerOptions {
@@ -23,25 +25,46 @@ export interface ServerOptions {
     model: Model
     connection: Pool
     dialect?: Dialect
-    subscriptions?: boolean
-    subscriptionPollInterval?: number
-    subscriptionConnection?: Pool
     graphiqlConsole?: boolean
     log?: Logger
     maxRequestSizeBytes?: number
+    maxRootFields?: number
+    maxResponseNodes?: number
+    subscriptions?: boolean
+    subscriptionPollInterval?: number
+    subscriptionConnection?: Pool
+    subscriptionMaxResponseNodes?: number
 }
 
 
 export async function serve(options: ServerOptions): Promise<ListeningServer> {
-    let {connection, subscriptionConnection, subscriptionPollInterval} = options
+    let {connection, subscriptionConnection, subscriptionPollInterval, maxResponseNodes, subscriptionMaxResponseNodes} = options
     let dialect = options.dialect ?? 'postgres'
 
     let schema = new SchemaBuilder(options).build()
+
     let context = () => {
+        let openreader: OpenreaderContext = new PoolOpenreaderContext(
+            dialect,
+            connection,
+            subscriptionConnection,
+            subscriptionPollInterval
+        )
+
+        if (maxResponseNodes) {
+            openreader.responseSizeLimit = new ResponseSizeLimit(maxResponseNodes)
+            openreader.subscriptionResponseSizeLimit = new ResponseSizeLimit(maxResponseNodes)
+        }
+
+        if (subscriptionMaxResponseNodes) {
+            openreader.subscriptionResponseSizeLimit = new ResponseSizeLimit(subscriptionMaxResponseNodes)
+        }
+
         return {
-            openreader: new PoolOpenreaderContext(dialect, connection, subscriptionConnection, subscriptionPollInterval)
+            openreader
         }
     }
+
     let disposals: Dispose[] = []
 
     return addServerCleanup(disposals, runApollo({
@@ -52,7 +75,8 @@ export async function serve(options: ServerOptions): Promise<ListeningServer> {
         subscriptions: options.subscriptions,
         log: options.log,
         graphiqlConsole: options.graphiqlConsole,
-        maxRequestSizeBytes: options.maxRequestSizeBytes
+        maxRequestSizeBytes: options.maxRequestSizeBytes,
+        maxRootFields: options.maxRootFields
     }), options.log)
 }
 
@@ -70,15 +94,21 @@ export interface ApolloOptions {
     graphiqlConsole?: boolean
     log?: Logger
     maxRequestSizeBytes?: number
+    maxRootFields?: number
 }
 
 
 export async function runApollo(options: ApolloOptions): Promise<ListeningServer> {
-    let {disposals, context, schema, log} = options
+    const {disposals, context, schema, log, maxRootFields} = options
+
     let maxRequestSizeBytes = options.maxRequestSizeBytes ?? 256 * 1024
     let app = express()
     let server = http.createServer(app)
 
+    const execute = maxRootFields
+            ? (args: ExecutionArgs) => executeWithLimit(maxRootFields, args)
+            : undefined
+
     if (options.subscriptions) {
         let wsServer = new WebSocketServer({
             server,
@@ -89,6 +119,7 @@ export async function runApollo(options: ApolloOptions): Promise<ListeningServer
             {
                 schema,
                 context,
+                execute,
                 onError(ctx, message, errors) {
                     if (log) {
                         // FIXME: we don't want to log client errors
@@ -111,6 +142,16 @@ export async function runApollo(options: ApolloOptions): Promise<ListeningServer
         schema,
         context,
         stopOnTerminationSignals: false,
+        executor: execute && (async req => {
+            return execute({
+                schema,
+                document: req.document,
+                rootValue: {},
+                contextValue: req.context,
+                variableValues: req.request.variables,
+                operationName: req.operationName
+            })
+        }),
         plugins: [
             ...options.plugins || [],
             {
@@ -136,7 +177,6 @@ export async function runApollo(options: ApolloOptions): Promise<ListeningServer
         setupGraphiqlConsole(app)
     }
 
-
     await apollo.start()
     disposals.push(() => apollo.stop())
 

package/src/sql/cursor.ts

@@ -154,7 +154,7 @@ export class EntityCursor implements Cursor {
             case "fk":
                 return new EntityCursor(
                     this.ctx,
-                    prop.type.foreignEntity,
+                    prop.type.entity,
                     {on: 'id', rhs: this.native(field)}
                 )
             case "lookup":
@@ -281,7 +281,7 @@ export class ObjectCursor implements Cursor {
             case "fk":
                 return new EntityCursor(
                     this.ctx,
-                    prop.type.foreignEntity,
+                    prop.type.entity,
                     {on: 'id', rhs: this.string(field)}
                 )
             default:

package/src/sql/mapping.ts

@@ -1,5 +1,5 @@
 import {unexpectedCase} from "@subsquid/util-internal"
-import {FieldRequest} from "../ir/fields"
+import {FieldRequest, FieldsByEntity} from '../ir/fields'
 
 
 export function mapRows(rows: any[][], fields: FieldRequest[]): any[] {
@@ -64,3 +64,20 @@ export function mapRow(row: any[], fields: FieldRequest[], ifType?: string): any
     }
     return rec
 }
+
+
+export function mapQueryableRows(rows: any[][], fields: FieldsByEntity): any[] {
+    let result = new Array(rows.length)
+    for (let i = 0; i < rows.length; i++) {
+        result[i] = mapQueryableRow(rows[i], fields)
+    }
+    return result
+}
+
+
+export function mapQueryableRow(row: any[], fields: FieldsByEntity): any {
+    let entity = row[0]
+    let rec = mapRow(row[1], fields[entity])
+    rec._isTypeOf = entity
+    return rec
+}

package/src/sql/printer.ts

@@ -1,14 +1,15 @@
-import {unexpectedCase} from "@subsquid/util-internal"
-import assert from "assert"
-import {Dialect} from "../dialect"
-import {EntityListArguments, OrderBy, Where} from "../ir/args"
-import {FieldRequest} from "../ir/fields"
-import {Model} from "../model"
-import {Cursor, EntityCursor} from "./cursor"
-import {AliasSet, ColumnSet, escapeIdentifier, JoinSet, printClause} from "./util"
+import {unexpectedCase} from '@subsquid/util-internal'
+import assert from 'assert'
+import {Dialect} from '../dialect'
+import {OrderBy, SortOrder, SqlArguments, Where} from '../ir/args'
+import {FieldRequest, FieldsByEntity} from '../ir/fields'
+import {Model} from '../model'
+import {getQueryableEntities} from '../model.tools'
+import {Cursor, EntityCursor} from './cursor'
+import {AliasSet, ColumnSet, escapeIdentifier, JoinSet, printClause} from './util'
 
 
-export class EntityListQueryPrinter {
+export class EntitySqlPrinter {
     private aliases: AliasSet
     private join: JoinSet
     private root: EntityCursor
@@ -19,9 +20,9 @@ export class EntityListQueryPrinter {
     constructor(
         private model: Model,
         private dialect: Dialect,
-        private entityName: string,
+        public readonly entityName: string,
         private params: unknown[],
-        private args: EntityListArguments = {},
+        private args: SqlArguments = {},
         fields?: FieldRequest[],
         aliases?: AliasSet
     ) {
@@ -43,12 +44,14 @@ export class EntityListQueryPrinter {
             this.populateWhere(this.root, args.where, this.where)
         }
         if (args.orderBy) {
-            this.populateOrderBy(this.root, args.orderBy)
+            this.traverseOrderBy(args.orderBy, (field, cursor, order) => {
+                this.orderBy.push(cursor.native(field) + ' ' + order)
+            })
         }
     }
 
-    private sub(entityName: string, args?: EntityListArguments, fields?: FieldRequest[]): EntityListQueryPrinter {
-        return new EntityListQueryPrinter(this.model, this.dialect, entityName, this.params, args, fields, this.aliases)
+    private sub(entityName: string, args?: SqlArguments, fields?: FieldRequest[]): EntitySqlPrinter {
+        return new EntitySqlPrinter(this.model, this.dialect, entityName, this.params, args, fields, this.aliases)
     }
 
     private populateColumns(cursor: Cursor, fields: FieldRequest[]): void {
@@ -128,7 +131,9 @@ export class EntityListQueryPrinter {
                 break
             }
             case "isNull": {
-                let f = cursor.ref(where.field)
+                let f = cursor.prop(where.field).type.kind == 'lookup'
+                    ? cursor.child(where.field).ref('id')
+                    : cursor.ref(where.field)
                 if (where.yes) {
                     exps.push(`${f} IS NULL`)
                 } else {
@@ -261,13 +266,17 @@ export class EntityListQueryPrinter {
         return printClause("AND", exps)
     }
 
-    private populateOrderBy(cursor: Cursor, orderBy: OrderBy): void {
+    traverseOrderBy(orderBy: OrderBy, cb: (field: string, cursor: Cursor, order: SortOrder) => void) {
+        this.visitOrderBy(this.root, orderBy, cb)
+    }
+
+    private visitOrderBy(cursor: Cursor, orderBy: OrderBy, cb: (field: string, cursor: Cursor, order: SortOrder) => void) {
         for (let field in orderBy) {
             let spec = orderBy[field]
             if (typeof spec == "string") {
-                this.orderBy.push(`${cursor.native(field)} ${spec}`)
+                cb(field, cursor, spec)
             } else {
-                this.populateOrderBy(cursor.child(field), spec)
+                this.visitOrderBy(cursor.child(field), spec, cb)
             }
         }
     }
@@ -280,14 +289,18 @@ export class EntityListQueryPrinter {
         return escapeIdentifier(this.dialect, name)
     }
 
-    addWhereDerivedFrom(field: string, parentIdExp: string): this {
+    private addWhereDerivedFrom(field: string, parentIdExp: string): this {
         this.where.push(`${this.root.native(field)} = ${parentIdExp}`)
         return this
     }
 
+    hasColumns(): boolean {
+        return this.columns.size() > 0
+    }
+
     printColumnList(options?: {withAliases?: boolean}): string {
+        assert(this.hasColumns())
         let names = this.columns.names()
-        assert(names.length > 0)
         if (options?.withAliases) {
             names = names.map((name, idx) => `${name} AS _c${idx}`)
         }
@@ -322,7 +335,110 @@ export class EntityListQueryPrinter {
         return `SELECT ${this.printColumnList({withAliases: true})} ${this.printFrom()}`
     }
 
-    printAsJsonRows(): string {
+    printAsCount(): string {
+        if (this.args.offset || this.args.limit) {
+            return `SELECT count(*) FROM (SELECT true ${this.printFrom()}) AS rows`
+        } else {
+            return `SELECT count(*) ${this.printFrom()}`
+        }
+    }
+
+    private printAsJsonRows(): string {
         return `SELECT ${this.printColumnListAsJsonArray()} AS row ${this.printFrom()}`
     }
 }
+
+
+export class QueryableSqlPrinter {
+    private printers: EntitySqlPrinter[] = []
+    private orders: SortOrder[] = []
+    private orderColumns: string[][] = []
+
+    constructor(
+        private model: Model,
+        private dialect: Dialect,
+        private queryableName: string,
+        private params: unknown[],
+        private args: SqlArguments = {},
+        fields?: FieldsByEntity
+    ) {
+        for (let entityName of getQueryableEntities(this.model, this.queryableName)) {
+            let entityFields = fields?.[entityName]
+
+            let printer = new EntitySqlPrinter(
+                model,
+                dialect,
+                entityName,
+                this.params,
+                {where: args.where},
+                entityFields
+            )
+
+            if (this.args.orderBy) {
+                let cols: string[] = []
+                this.orders.length = 0
+                printer.traverseOrderBy(this.args.orderBy, (field, cursor, order) => {
+                    let col = field == '_type' ? `'${entityName}'` : cursor.native(field)
+                    this.orders.push(order)
+                    cols.push(`${col} AS o${this.orders.length}`)
+                })
+                this.orderColumns.push(cols)
+            }
+
+            this.printers.push(printer)
+        }
+    }
+
+    print(): string {
+        let from = this.printers.map((printer, idx) => {
+            let cols: string[] = []
+            cols.push(`'${printer.entityName}' AS e`)
+            if (printer.hasColumns()) {
+                cols.push(printer.printColumnListAsJsonArray() + ' AS d')
+            } else {
+                cols.push('null AS d')
+            }
+            cols.push(...this.orderColumns[idx])
+            return `SELECT ${cols.join(', ')} ${printer.printFrom()}`
+        }).join('\nUNION ALL\n')
+
+        let args = this.printArgs()
+        if (args) {
+            return `SELECT e, d FROM (\n${from}\n) AS rows` + args
+        } else {
+            return from
+        }
+    }
+
+    printAsCount(): string {
+        let union = this.orders.length
+            ? this.printers.map((printer, idx) => {
+                return `SELECT ${this.orderColumns[idx].join(', ')} ${printer.printFrom()}`
+            })
+            : this.printers.map(printer => {
+                return `SELECT true ${printer.printFrom()}`
+            })
+
+        let from = union.join('\nUNION ALL\n')
+        let args = this.printArgs()
+        if (args) {
+            from = `SELECT true FROM (\n${from}\n) AS src` + args
+        }
+
+        return `SELECT count(*) FROM (\n${from}\n) AS rows`
+    }
+
+    private printArgs(): string {
+        let sql = ''
+        if (this.orders.length) {
+            sql += '\nORDER BY ' + this.orders.map((o, idx) => `o${idx + 1} ${o}`).join(', ')
+        }
+        if (this.args.offset) {
+            sql += `\nOFFSET ${this.args.offset}`
+        }
+        if (this.args.limit) {
+            sql += `\nLIMIT ${this.args.limit}`
+        }
+        return sql
+    }
+}