@subsquid/openreader 1.0.2 → 2.1.0

package/src/sql/query.ts

@@ -0,0 +1,194 @@
+import {assertNotNull} from "@subsquid/util-internal"
+import assert from "assert"
+import type {Dialect} from "../dialect"
+import type {EntityListArguments, Where} from "../ir/args"
+import {
+    decodeRelayConnectionCursor,
+    encodeRelayConnectionCursor,
+    RelayConnectionEdge,
+    RelayConnectionPageInfo,
+    RelayConnectionRequest,
+    RelayConnectionResponse
+} from "../ir/connection"
+import type {FieldRequest} from "../ir/fields"
+import type {Model} from "../model"
+import {toSafeInteger} from "../util/util"
+import {mapRows} from "./mapping"
+import {EntityListQueryPrinter} from "./printer"
+
+
+export interface Query<T> {
+    readonly sql: string
+    readonly params: unknown[]
+    map(rows: any[][]): T
+}
+
+
+export class EntityListQuery implements Query<any[]> {
+    public readonly sql: string
+    public readonly params: unknown[] = []
+
+    constructor(
+        model: Model,
+        dialect: Dialect,
+        entityName: string,
+        private fields: FieldRequest[],
+        args: EntityListArguments
+    ) {
+        this.sql = new EntityListQueryPrinter(model, dialect, entityName, this.params, args, fields).print()
+    }
+
+    map(rows: any[][]): any[] {
+        return mapRows(rows, this.fields)
+    }
+}
+
+
+export class EntityByIdQuery {
+    public readonly sql: string
+    public readonly params: unknown[] = []
+
+    constructor(
+        model: Model,
+        dialect: Dialect,
+        entityName: string,
+        private fields: FieldRequest[],
+        id: string
+    ) {
+        this.sql = new EntityListQueryPrinter(
+            model,
+            dialect,
+            entityName,
+            this.params,
+            {where: {op: 'eq', field: 'id', value: id}},
+            fields
+        ).print()
+    }
+
+    map(rows: any[][]): any {
+        assert(rows.length < 2)
+        return mapRows(rows, this.fields)[0]
+    }
+}
+
+
+export class EntityCountQuery implements Query<number> {
+    public readonly sql: string
+    public readonly params: unknown[] = []
+
+    constructor(
+        model: Model,
+        dialect: Dialect,
+        entityName: string,
+        where?: Where
+    ) {
+        this.sql = 'SELECT count(*) ' + new EntityListQueryPrinter(model, dialect, entityName, this.params, {where}).printFrom()
+    }
+
+    map(rows: any[][]): number {
+        return toCount(rows)
+    }
+}
+
+
+export class EntityConnectionQuery implements Query<RelayConnectionResponse> {
+    public readonly sql: string
+    public readonly params: unknown[] = []
+    private offset = 0
+    private limit = 100
+    private edgeNode?: FieldRequest[]
+    private edgeCursor?: boolean
+    private pageInfo?: boolean
+    private totalCount?: boolean
+
+    constructor(
+        model: Model,
+        dialect: Dialect,
+        entityName: string,
+        req: RelayConnectionRequest
+    ) {
+        this.setOffsetAndLimit(req)
+        this.edgeCursor = req.edgeCursor
+        this.pageInfo = req.pageInfo
+        this.totalCount = req.totalCount
+
+        let printer = new EntityListQueryPrinter(model, dialect, entityName, this.params, {
+            orderBy: req.orderBy,
+            where: req.where,
+            offset: this.offset,
+            limit: this.limit + 1
+        }, req.edgeNode)
+
+        if (req.edgeNode?.length) {
+            this.edgeNode = req.edgeNode
+            this.sql = printer.print()
+        } else {
+            this.sql = `SELECT count(*) FROM (SELECT true ${printer.printFrom()}) AS rows`
+        }
+    }
+
+    private setOffsetAndLimit(req: RelayConnectionRequest): void {
+        if (req.after != null) {
+            this.offset = assertNotNull(decodeRelayConnectionCursor(req.after))
+        }
+        if (req.first != null) {
+            assert(req.first >= 0)
+            this.limit = req.first
+        }
+    }
+
+    map(rows: any[][]): RelayConnectionResponse {
+        let res: RelayConnectionResponse = {}
+        if (this.edgeNode) {
+            let nodes = mapRows(rows, this.edgeNode)
+            let edges: RelayConnectionEdge[] = new Array(Math.min(this.limit, nodes.length))
+            for (let i = 0; i < edges.length; i++) {
+                edges[i] = {
+                    node: nodes[i],
+                    cursor: this.edgeCursor ? encodeRelayConnectionCursor(this.offset + i + 1) : undefined
+                }
+            }
+            res.edges = edges
+            res.pageInfo = this.getPageInfo(nodes.length)
+            res.totalCount = this.getTotalCount(nodes.length)
+        } else {
+            let count = toCount(rows)
+            if (this.edgeCursor) {
+                res.edges = new Array(Math.min(this.limit, count))
+                for (let i = 0; i < res.edges.length; i++) {
+                    res.edges[i] = {
+                        cursor: encodeRelayConnectionCursor(this.offset + i + 1)
+                    }
+                }
+            }
+            res.pageInfo = this.getPageInfo(count)
+            res.totalCount = this.getTotalCount(count)
+        }
+        return res
+    }
+
+    private getPageInfo(count: number): Partial<RelayConnectionPageInfo> | undefined {
+        if (!this.pageInfo) return
+        return {
+            hasNextPage: count > this.limit,
+            hasPreviousPage: count > 0 && this.offset > 0,
+            startCursor: count > 0 ? encodeRelayConnectionCursor(this.offset + 1) : '',
+            endCursor: count > 0 ? encodeRelayConnectionCursor(this.offset + Math.min(this.limit, count)) : ''
+        }
+    }
+
+    private getTotalCount(count: number): number | undefined {
+        if (!this.totalCount) return
+        if (count > 0 && count <= this.limit) {
+            return this.offset + count
+        } else {
+            return undefined
+        }
+    }
+}
+
+
+function toCount(rows: any[][]): number {
+    assert(rows.length == 1)
+    return toSafeInteger(rows[0][0])
+}

package/src/sql/util.ts

@@ -0,0 +1,89 @@
+import type {Dialect} from "../dialect"
+
+
+export function escapeIdentifier(dialect: Dialect, name: string): string {
+    return `"${name.replace(/"/g, '""')}"`
+}
+
+
+export class ColumnSet {
+    private columns: Map<string, number> = new Map()
+
+    add(column: string): number {
+        let idx = this.columns.get(column)
+        if (idx == null) {
+            idx = this.columns.size
+            this.columns.set(column, idx)
+        }
+        return idx
+    }
+
+    names(): string[] {
+        return [...this.columns.keys()]
+    }
+
+    size(): number {
+        return this.columns.size
+    }
+}
+
+
+/**
+ * LEFT OUTER JOIN "{table}" "{alias}" ON "{alias}"."{column}" = {rhs}
+ */
+export interface Join {
+    table: string
+    alias: string
+    column: string
+    rhs: string
+}
+
+
+export class JoinSet {
+    private joins: Map<string, Join> = new Map()
+
+    constructor(private aliases: AliasSet) {
+    }
+
+    add(table: string, column: string, rhs: string): string {
+        let key = `${table} ${column} ${rhs}`
+        let e = this.joins.get(key)
+        if (!e) {
+            e = {
+                table,
+                alias: this.aliases.add(table),
+                column,
+                rhs
+            }
+            this.joins.set(key, e)
+        }
+        return e.alias
+    }
+
+    forEach(cb: (join: Join) => void): void {
+        this.joins.forEach(join => cb(join))
+    }
+}
+
+
+export class AliasSet {
+    private aliases: Record<string, number> = {}
+
+    add(name: string): string {
+        if (this.aliases[name]) {
+            return name + "_" + (this.aliases[name]++)
+        } else {
+            this.aliases[name] = 1
+            return name
+        }
+    }
+}
+
+
+export function printClause(op: string, exps: string[]): string {
+    switch(exps.length) {
+        case 0: return ''
+        case 1: return exps[0]
+        default: return exps.join(' ' + op + ' ')
+    }
+}

package/src/subscription.ts

@@ -0,0 +1,46 @@
+import deepEqual from "deep-equal"
+
+
+export class Subscription<T> implements AsyncIterator<T>, AsyncIterable<T> {
+    private timer?: NodeJS.Timer
+    private prev?: T
+    private hasNoVal = true
+
+    constructor(private interval: number, private poll: () => Promise<T>) {}
+
+    [Symbol.asyncIterator]() {
+        return this
+    }
+
+    async next() {
+        if (this.hasNoVal) {
+            this.prev = await this.poll()
+            this.hasNoVal = false
+        }
+        let value
+        do {
+            await new Promise(resolve => {
+                this.timer = setTimeout(resolve, this.interval)
+            })
+            value = await this.poll()
+        } while (deepEqual(this.prev, value))
+        this.prev = value
+        return {done: false, value}
+    }
+
+    async return() {
+        if (this.timer != null) {
+            clearTimeout(this.timer)
+            this.timer = undefined
+        }
+        return EOS
+    }
+}
+
+
+const EOS = {
+    done: true,
+    get value(): any {
+        throw new Error('Unexpected value access')
+    }
+}

package/src/test/fts.test.ts

@@ -6,7 +6,7 @@ function tsvector(columns: string[]) {
 }
 
 
-isCockroach() || describe('full text search', function () {
+isCockroach() || describe.skip('full text search', function () {
     useDatabase([
         `create table foo (
             id text primary key, 

package/src/test/limits.test.ts

@@ -0,0 +1,163 @@
+import expect from 'expect'
+import {useDatabase, useServer} from './setup'
+
+
+describe('response size limits', function() {
+    useDatabase([
+        `create table "order1" (id text primary key)`,
+        `create table item1 (id text primary key, order_id text, name text)`,
+        `create table "order2" (id text primary key)`,
+        `create table item2 (id text primary key, order_id text, name text)`,
+        `create table "order3" (id text primary key)`,
+        `create table item3 (id text primary key, order_id text, name text)`,
+    ])
+
+    const client = useServer(`
+        type Order1 @entity {
+            id: ID!
+            items: [Item1!]! @derivedFrom(field: "order")
+        }
+        
+        type Item1 @entity {
+            id: ID!
+            order: Order1!
+            name: String
+        }
+        
+        type Order2 @entity @cardinality(value: 10) {
+            id: ID!
+            items: [Item2!]! @derivedFrom(field: "order")
+        }
+        
+        type Item2 @entity {
+            id: ID!
+            order: Order2!
+            name: String
+        }
+        
+        type Order3 @entity {
+            id: ID!
+            items: [Item3!]! @derivedFrom(field: "order") @cardinality(value: 10)
+        }
+        
+        type Item3 @entity {
+            id: ID!
+            order: Order3!
+            name: String @byteWeight(value: 10.0)
+        }
+    `, {
+        maxResponseNodes: 50,
+        maxRootFields: 3
+    })
+
+    it('unlimited requests fail', async function() {
+        let result = await client.query(`
+            query {
+                order1s {
+                    id
+                }
+            }
+        `)
+        expect(result).toMatchObject({
+            data: null,
+            errors: [
+                expect.objectContaining({message: 'response might exceed the size limit', path: ['order1s']})
+            ]
+        })
+    })
+
+    it('limited requests work', function() {
+        return client.test(`
+            query {
+                order1s(limit: 10) {
+                    items(limit: 2) {
+                        id
+                    }
+                }
+            }
+        `, {
+            order1s: []
+        })
+    })
+
+    it('entity level cardinalities are respected', function() {
+        return client.test(`
+            query {
+                order2s {
+                    id
+                }
+            }
+        `, {
+            order2s: []
+        })
+    })
+
+    it('item cardinalities are respected', function() {
+        return client.test(`
+            query {
+                order3s(limit: 1) {
+                    items { id }
+                }
+            }
+        `, {
+            order3s: []
+        })
+    })
+
+    it('@byteWeight annotations are respected', async function() {
+        let result = await client.query(`
+            query {
+                order3s(limit: 1) {
+                    items(limit: 8) { name }
+                }
+            }
+        `)
+        expect(result).toEqual({
+            data: null,
+            errors: [
+                expect.objectContaining({
+                    message: 'response might exceed the size limit',
+                    path: ['order3s']
+                })
+            ]
+        })
+        await client.test(`
+            query {
+                order3s(limit: 1) {
+                    items(limit: 4) { name }
+                }
+            }
+        `, {
+            order3s: []
+        })
+    })
+
+    it('id_in conditions are understood', function() {
+        return client.test(`
+            query {
+                order1s(where: {id_in: ["1", "2", "3"]}) {
+                    id
+                }
+            }
+        `, {
+            order1s: []
+        })
+    })
+
+    it('root query fields limit', async function() {
+        return client.errorTest(`
+            query {
+                a: order1ById(id: "1") { id }
+                b: order1ById(id: "1") { id }
+                c: order1ById(id: "1") { id }
+                d: order1ById(id: "1") { id }
+            }
+        `, {
+            errors: [
+                expect.objectContaining({
+                    message: 'only 3 root query fields allowed, but got 4'
+                })
+            ]
+        })
+    })
+})

package/src/test/setup.ts

@@ -1,9 +1,10 @@
 import {assertNotNull} from "@subsquid/util-internal"
+import {ListeningServer} from "@subsquid/util-internal-http-server"
 import {Client} from "gql-test-client"
 import {parse} from "graphql"
 import {Client as PgClient, ClientBase, Pool} from "pg"
-import {buildModel, buildSchema} from "../gql/schema"
-import {ListeningServer, serve} from "../server"
+import {buildModel, buildSchema} from "../model.schema"
+import {serve, ServerOptions} from '../server'
 
 
 export function isCockroach(): boolean {
@@ -22,7 +23,7 @@ export const db_config = {
 }
 
 
-async function withClient(block: (client: ClientBase) => Promise<void>): Promise<void> {
+export async function withDatabase(block: (client: ClientBase) => Promise<void>): Promise<void> {
     let client = new PgClient(db_config)
     await client.connect()
     try {
@@ -33,8 +34,8 @@ async function withClient(block: (client: ClientBase) => Promise<void>): Promise
 }
 
 
-export function databaseInit(sql: string[]): Promise<void> {
-    return withClient(async client => {
+export function databaseExecute(sql: string[]): Promise<void> {
+    return withDatabase(async client => {
         for (let i = 0; i < sql.length; i++) {
             await client.query(sql[i])
         }
@@ -43,7 +44,7 @@ export function databaseInit(sql: string[]): Promise<void> {
 
 
 export function databaseDelete(): Promise<void> {
-    return withClient(async client => {
+    return withDatabase(async client => {
         await client.query(`DROP SCHEMA IF EXISTS root CASCADE`)
         await client.query(`CREATE SCHEMA root`)
     })
@@ -53,25 +54,29 @@ export function databaseDelete(): Promise<void> {
 export function useDatabase(sql: string[]): void {
     before(async () => {
         await databaseDelete()
-        await databaseInit(sql)
+        await databaseExecute(sql)
     })
 }
 
 
-export function useServer(schema: string): Client {
+export function useServer(schema: string, options?: Partial<ServerOptions>): Client {
     let client = new Client('not defined')
     let db = new Pool(db_config)
     let info: ListeningServer | undefined
     before(async () => {
         info = await serve({
-            db,
+            connection: db,
             model: buildModel(buildSchema(parse(schema))),
             port: 0,
-            dialect: isCockroach() ? 'cockroach' : 'postgres'
+            dialect: isCockroach() ? 'cockroach' : 'postgres',
+            subscriptions: true,
+            subscriptionPollInterval: 500,
+            maxRootFields: 10,
+            ...options
         })
         client.endpoint = `http://localhost:${info.port}/graphql`
     })
-    after(() => info?.stop())
+    after(() => info?.close())
     after(() => db.end())
     return client
 }

package/src/test/subscription.test.ts

@@ -0,0 +1,98 @@
+import {wait} from "@subsquid/util-internal"
+import expect from "expect"
+import {databaseExecute, useDatabase, useServer} from "./setup"
+
+
+describe("subscriptions", function() {
+    useDatabase([
+        `create table "order" (id text primary key, kind text not null, name text)`,
+        `create table item (id text primary key, order_id text, name text)`,
+    ])
+
+    const client = useServer(`
+        type Order @entity {
+            id: ID!
+            kind: String!
+            name: String
+            items: [Item!]! @derivedFrom(field: "order")
+        }
+        
+        type Item @entity {
+            id: ID!
+            order: Order
+            name: String
+        }
+    `)
+
+    it("entity list", function() {
+        return client.subscriptionTest(`
+            subscription {
+                orders(where: {kind_eq: "list"}, orderBy: id_ASC) {
+                    id
+                    name
+                    items(orderBy: id_ASC) {
+                        name
+                    }
+                }
+            }
+        `, async take => {
+            await wait(1000)
+            await databaseExecute([
+                `insert into "order" (id, kind) values ('1', 'list')`,
+                `insert into "order" (id, kind) values ('2', 'foo')`,
+            ])
+            expect(await take()).toEqual({
+                data: {
+                    orders: [{id: '1', name: null, items: []}]
+                }
+            })
+            await databaseExecute([`
+                update "order" set name = 'hello' where id in ('1', '2');
+                insert into item (id, "order_id", name) values ('1-1', '1', '123')
+            `])
+            expect(await take()).toEqual({
+                data: {
+                    orders: [
+                        {id: '1', name: 'hello', items: [{name: '123'}]}
+                    ]
+                }
+            })
+        })
+    })
+
+    it("entity by id", async function() {
+        await databaseExecute([
+            `insert into "order" (id, kind) values ('3', 'by id')`,
+            `insert into item (id, "order_id", name) values ('3-1', '3', 'hello')`
+        ])
+        await client.subscriptionTest(`
+            subscription {
+                third: orderById(id: "3") {
+                    name
+                    items(orderBy: id_ASC) {
+                        name
+                    }
+                }
+            }
+        `, async take => {
+            await wait(1000)
+            await databaseExecute([`
+                start transaction;
+                update "order" set name = 'foo' where id = '3';
+                insert into item (id, "order_id", name) values ('3-2', '3', 'world');
+                commit;
+            `])
+            expect(await take()).toEqual({
+                data: {
+                    third: {
+                        name: 'foo',
+                        items: [
+                            {name: 'hello'},
+                            {name: 'world'}
+                        ]
+                    }
+                }
+            })
+        })
+    })
+})

package/src/tools.ts

@@ -3,7 +3,7 @@ import * as fs from "fs"
 import * as path from "path"
 import {parse, Source} from "graphql"
 import process from "process"
-import {buildModel, buildSchema} from "./gql/schema"
+import {buildModel, buildSchema} from "./model.schema"
 import type {Model} from "./model"