grom_native 0.1.0

data/ext/net/spec/methods_test.go

@@ -0,0 +1,164 @@
+package spec
+
+import (
+	"errors"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/ukparliament/gromnative/ext/net"
+	. "github.com/ukparliament/gromnative/ext/types/net"
+	"gopkg.in/jarcoal/httpmock.v1"
+	"net/http"
+)
+
+type errReader int
+
+func (errReader) Read(p []byte) (n int, err error) { return 0, errors.New("test error") }
+func (errReader) Close() error                     { return nil }
+
+var _ = Describe("Net", func() {
+	Describe("Get", func() {
+		Context("with a valid URI", func() {
+			BeforeEach(func() {
+				httpmock.RegisterResponder("GET", "https://api.parliament.uk/query/person_by_id?person_id=43RHonMf",
+					httpmock.NewStringResponder(200, `<https://id.parliament.uk/43RHonMf> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://id.parliament.uk/schema/Person> .\n\r<https://id.parliament.uk/43RHonMf> <https://id.parliament.uk/schema/personGivenName> "Diane" .\n\r<https://id.parliament.uk/43RHonMf> <https://id.parliament.uk/schema/personOtherNames> "Julie" .\n\r<https://id.parliament.uk/43RHonMf> <https://id.parliament.uk/schema/personFamilyName> "Abbott" .\n\r<https://id.parliament.uk/43RHonMf> <https://id.parliament.uk/schema/oppositionPersonHasOppositionIncumbency> <https://id.parliament.uk/wE8Hq016> .\n\r<https://id.parliament.uk/wE8Hq016> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://id.parliament.uk/schema/OppositionIncumbency> .\n\r<https://id.parliament.uk/wE8Hq016> <https://id.parliament.uk/schema/incumbencyStartDate> "2016-06-27+01:00"^^<http://www.w3.org/2001/XMLSchema#date> .`))
+			})
+
+			It("makes the expected request", func() {
+				resp, err := net.Get(&GetInput{Uri: "https://api.parliament.uk/query/person_by_id?person_id=43RHonMf"})
+
+				Expect(resp).To(Equal(&GetOutput{
+					Uri:        "https://api.parliament.uk/query/person_by_id?person_id=43RHonMf",
+					Body:       []byte("<https://id.parliament.uk/43RHonMf> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://id.parliament.uk/schema/Person> .\\n\\r<https://id.parliament.uk/43RHonMf> <https://id.parliament.uk/schema/personGivenName> \"Diane\" .\\n\\r<https://id.parliament.uk/43RHonMf> <https://id.parliament.uk/schema/personOtherNames> \"Julie\" .\\n\\r<https://id.parliament.uk/43RHonMf> <https://id.parliament.uk/schema/personFamilyName> \"Abbott\" .\\n\\r<https://id.parliament.uk/43RHonMf> <https://id.parliament.uk/schema/oppositionPersonHasOppositionIncumbency> <https://id.parliament.uk/wE8Hq016> .\\n\\r<https://id.parliament.uk/wE8Hq016> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://id.parliament.uk/schema/OppositionIncumbency> .\\n\\r<https://id.parliament.uk/wE8Hq016> <https://id.parliament.uk/schema/incumbencyStartDate> \"2016-06-27+01:00\"^^<http://www.w3.org/2001/XMLSchema#date> ."),
+					StatusCode: 200,
+				}))
+				Expect(err).NotTo(HaveOccurred())
+			})
+
+			Context("with headers", func() {
+				It("includes the headers in a request", func() {
+					httpmock.RegisterResponder(
+						"GET",
+						"https://api.parliament.uk/query/person_by_id?person_id=43RHonMf",
+						func(req *http.Request) (*http.Response, error) {
+							httpHeaders := http.Header{}
+							httpHeaders.Add("Foo", "Bar")
+							httpHeaders.Add("Bar", "Baz")
+
+							Expect(req.Header).To(Equal(httpHeaders))
+
+							return httpmock.NewStringResponse(200, "done"), nil
+						},
+					)
+
+					var headers []*GetInput_Header
+					headers = append(headers, &GetInput_Header{Key: "Foo", Value: "Bar"}, &GetInput_Header{Key: "Bar", Value: "Baz"})
+					resp, err := net.Get(&GetInput{Uri: "https://api.parliament.uk/query/person_by_id?person_id=43RHonMf", Headers: headers})
+
+					Expect(resp).To(Equal(&GetOutput{
+						Uri:        "https://api.parliament.uk/query/person_by_id?person_id=43RHonMf",
+						Body:       []byte("done"),
+						StatusCode: 200,
+					}))
+					Expect(err).NotTo(HaveOccurred())
+				})
+			})
+
+			Context("with an error making the request", func() {
+				BeforeEach(func() {
+					httpmock.RegisterResponder(
+						"GET",
+						"https://api.parliament.uk/query/person_by_id?person_id=43RHonMf",
+						func(req *http.Request) (*http.Response, error) {
+							return httpmock.NewStringResponse(200, "done"), errors.New("There was a problem")
+						},
+					)
+				})
+
+				It("returns the expected response and error", func() {
+					resp, err := net.Get(&GetInput{Uri: "https://api.parliament.uk/query/person_by_id?person_id=43RHonMf"})
+
+					Expect(resp).To(Equal(&GetOutput{
+						Uri:   "https://api.parliament.uk/query/person_by_id?person_id=43RHonMf",
+						Error: "Get https://api.parliament.uk/query/person_by_id?person_id=43RHonMf: There was a problem",
+					}))
+
+					Expect(err).To(HaveOccurred())
+					Expect(err.Error()).To(Equal("Get https://api.parliament.uk/query/person_by_id?person_id=43RHonMf: There was a problem"))
+				})
+			})
+
+			Context("with an error reading the response body", func() {
+				BeforeEach(func() {
+					httpmock.RegisterResponder(
+						"GET",
+						"https://api.parliament.uk/query/person_by_id?person_id=43RHonMf",
+						func(req *http.Request) (*http.Response, error) {
+							return &http.Response{StatusCode: 200, Body: errReader(0)}, nil
+						},
+					)
+				})
+
+				It("returns an error", func() {
+					resp, err := net.Get(&GetInput{Uri: "https://api.parliament.uk/query/person_by_id?person_id=43RHonMf"})
+
+					Expect(resp).To(Equal(&GetOutput{
+						Uri:        "https://api.parliament.uk/query/person_by_id?person_id=43RHonMf",
+						StatusCode: int32(200),
+						Error:      "Error reading body from https://api.parliament.uk/query/person_by_id?person_id=43RHonMf: test error",
+					}))
+
+					Expect(err).To(HaveOccurred())
+					Expect(err.Error()).To(Equal("test error"))
+				})
+			})
+
+			Context("with a non-200 status code", func() {
+				BeforeEach(func() {
+					httpmock.RegisterResponder(
+						"GET",
+						"https://api.parliament.uk/query/person_by_id?person_id=43RHonMf",
+						func(req *http.Request) (*http.Response, error) {
+							return httpmock.NewStringResponse(500, "Error"), nil
+						},
+					)
+				})
+
+				It("returns an error", func() {
+					resp, err := net.Get(&GetInput{Uri: "https://api.parliament.uk/query/person_by_id?person_id=43RHonMf"})
+
+					Expect(resp).To(Equal(&GetOutput{
+						Uri:        "https://api.parliament.uk/query/person_by_id?person_id=43RHonMf",
+						Body:       []byte("Error"),
+						StatusCode: int32(500),
+						Error:      "Received 500 status code from https://api.parliament.uk/query/person_by_id?person_id=43RHonMf: Error",
+					}))
+
+					Expect(err).To(HaveOccurred())
+					Expect(err.Error()).To(Equal("Received 500 status code from https://api.parliament.uk/query/person_by_id?person_id=43RHonMf: Error"))
+				})
+			})
+		})
+
+		Context("with an invalid URI", func() {
+			BeforeEach(func() {
+				httpmock.DeactivateAndReset()
+			})
+
+			AfterEach(func() {
+				httpmock.Activate()
+			})
+
+			It("returns the expected values", func() {
+				uri := "some_invalid-value.foo"
+				errorString := "Get some_invalid-value.foo: unsupported protocol scheme \"\""
+
+				resp, err := net.Get(&GetInput{Uri: uri})
+
+				Expect(resp).To(Equal(&GetOutput{Uri: "some_invalid-value.foo", Error: errorString}))
+
+				Expect(err).To(HaveOccurred())
+				Expect(err.Error()).To(Equal(errorString))
+			})
+		})
+	})
+})

data/ext/net/spec/net_suite_test.go

@@ -0,0 +1,31 @@
+package spec
+
+import (
+  "gopkg.in/jarcoal/httpmock.v1"
+	"io/ioutil"
+	"log"
+	"testing"
+
+  . "github.com/onsi/ginkgo"
+  . "github.com/onsi/gomega"
+)
+
+func TestNet(t *testing.T) {
+	log.SetOutput(ioutil.Discard)
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Net Suite")
+}
+
+var _ = BeforeSuite(func() {
+	// block all HTTP requests
+	httpmock.Activate()
+})
+
+var _ = BeforeEach(func() {
+	// remove any mocks
+	httpmock.Reset()
+})
+
+var _ = AfterSuite(func() {
+	httpmock.DeactivateAndReset()
+})

data/ext/processor/processor.go

@@ -0,0 +1,104 @@
+package processor
+
+import (
+	"bytes"
+	"strings"
+	"github.com/wallix/triplestore"
+	"log"
+)
+
+type Triple struct {
+	Subject   string `json:"subject"`
+	Predicate string `json:"predicate"`
+	Object    string `json:"object"`
+}
+
+type ProcessorInput struct {
+	Body []byte
+}
+
+type ProcessorOutput struct {
+	StatementsBySubject map[string][]Triple
+	EdgesBySubject      map[string]map[string][]string
+	Error               string
+}
+
+func NewTriple(t triplestore.Triple) Triple {
+	object := ""
+
+	bnode, isBnode := t.Object().Bnode()
+	literalObj, isLiteral := t.Object().Literal()
+	resource, _ := t.Object().Resource()
+
+	if isBnode {
+		object = "_:" + bnode
+	} else if isLiteral {
+		var literal string
+
+		if literalObj.Lang() != "" {
+			literal = "\"" + literalObj.Value() + "\"@" + literalObj.Lang()
+		} else {
+			literal = "\"" + literalObj.Value() + "\"^^<" + string(literalObj.Type()) + ">"
+		}
+
+		object = literal
+	} else {
+		object = "<" + resource + ">"
+	}
+
+	return Triple{
+		Subject:   t.Subject(),
+		Predicate: t.Predicate(),
+		Object:    object,
+	}
+}
+
+func Process(input *ProcessorInput) (*ProcessorOutput, error) {
+	output := ProcessorOutput{}
+
+	// Used to group all statements under a shared subject
+	statementsBySubject := make(map[string][]Triple)
+	// Used to show how one object, through a predicate, links to one or more object
+	edgesBySubject := make(map[string]map[string][]string)
+
+	log.Println("Decoding")
+	dec := triplestore.NewDatasetDecoder(triplestore.NewLenientNTDecoder, bytes.NewReader(input.Body))
+	tris, err := dec.Decode()
+	if err != nil {
+		log.Printf("Error decoding: %v\n", err)
+		output.Error = err.Error()
+		return &output, err
+	}
+	log.Printf("Decoded %v triples", len(tris))
+
+	for i := 0; i < len(tris); i++ {
+		triple := tris[i]
+
+		subject := triple.Subject()
+		predicate := triple.Predicate()
+		object := triple.Object()
+
+		statementsBySubject[subject] = append(statementsBySubject[subject], NewTriple(triple))
+
+		// decide if this is an edge
+		objectResource, _ := object.Resource()
+		if objectResource != "" && predicate != "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" {
+			if edgesBySubject[subject] == nil {
+				edgesBySubject[subject] = make(map[string][]string)
+			}
+
+			predicateSlice := strings.Split(predicate, "/")
+			predicateObject := predicateSlice[len(predicateSlice)-1]
+
+			edgesBySubject[subject][predicateObject] = append(edgesBySubject[subject][predicateObject], objectResource)
+		}
+	}
+
+	log.Printf("Found %v subjects\n", len(statementsBySubject))
+
+	// Pass our statements and edges back in our response
+	output.StatementsBySubject = statementsBySubject
+	output.EdgesBySubject = edgesBySubject
+
+	return &output, nil
+}

data/ext/processor/spec/methods_test.go

@@ -0,0 +1,102 @@
+package spec
+
+import (
+  . "github.com/onsi/ginkgo"
+  . "github.com/onsi/gomega"
+  "github.com/ukparliament/gromnative/ext/processor"
+  "github.com/wallix/triplestore"
+  "io/ioutil"
+)
+
+var _ = Describe("Processor", func() {
+  Describe("NetTriple", func() {
+    Context("object is a BNode", func() {
+      It("creates the expected triple", func() {
+        expected := processor.Triple{
+          Subject: "https://id.parliament.uk/12345678",
+          Predicate: "https://id.parliament.uk/shema/PredicateName",
+          Object: "_:node39387803",
+        }
+
+        triple := triplestore.SubjPredBnode("https://id.parliament.uk/12345678", "https://id.parliament.uk/shema/PredicateName", "node39387803")
+
+        result := processor.NewTriple(triple)
+
+        Expect(result).To(Equal(expected))
+      })
+    })
+
+    Context("object is a Literal", func() {
+      It("creates the expected triple", func() {
+        expected := processor.Triple{
+          Subject: "https://id.parliament.uk/12345678",
+          Predicate: "https://id.parliament.uk/shema/PredicateName",
+          Object:  "\"12\"^^<xsd:integer>",
+        }
+
+        triple, err := triplestore.SubjPredLit("https://id.parliament.uk/12345678", "https://id.parliament.uk/shema/PredicateName", 12)
+
+        result := processor.NewTriple(triple)
+
+        Expect(result).To(Equal(expected))
+        Expect(err).NotTo(HaveOccurred())
+      })
+    })
+
+    Context("object is a Resource", func() {
+      It("creates the expected triple", func() {
+        expected := processor.Triple{
+          Subject: "https://id.parliament.uk/12345678",
+          Predicate: "https://id.parliament.uk/shema/PredicateName",
+          Object:  "<https://id.parliament.uk/23456789>",
+        }
+
+        triple := triplestore.SubjPredRes("https://id.parliament.uk/12345678", "https://id.parliament.uk/shema/PredicateName", "https://id.parliament.uk/23456789")
+
+        result := processor.NewTriple(triple)
+
+        Expect(result).To(Equal(expected))
+      })
+    })
+  })
+
+  Describe("Process", func() {
+    Context("with no edges", func() {
+      It("returns empty edges object", func() {
+        fixture, _ := ioutil.ReadFile("../../../spec/fixtures/no_edges.nt")
+
+        res, err := processor.Process(&processor.ProcessorInput{ Body: fixture })
+
+        Expect(len(res.StatementsBySubject)).To(Equal(1))
+        Expect(len(res.EdgesBySubject)).To(Equal(0))
+        Expect(err).NotTo(HaveOccurred())
+      })
+    })
+
+    Context("with empty data", func() {
+      It("returns empty objects", func() {
+        expected := &processor.ProcessorOutput{
+          StatementsBySubject: make(map[string][]processor.Triple),
+          EdgesBySubject: make(map[string]map[string][]string),
+        }
+
+        res, err := processor.Process(&processor.ProcessorInput{ Body: []byte("") })
+
+        Expect(res).To(Equal(expected))
+        Expect(err).NotTo(HaveOccurred())
+      })
+    })
+
+    Context("with full data", func() {
+      It("returns the expected objects", func() {
+        fixture, _ := ioutil.ReadFile("../../../spec/fixtures/full.nt")
+
+        res, err := processor.Process(&processor.ProcessorInput{ Body: fixture })
+
+        Expect(len(res.StatementsBySubject)).To(Equal(43))
+        Expect(len(res.EdgesBySubject)).To(Equal(26))
+        Expect(err).NotTo(HaveOccurred())
+      })
+    })
+  })
+})

data/ext/processor/spec/processor_suite_test.go

@@ -0,0 +1,16 @@
+package spec
+
+import (
+  "io/ioutil"
+  "log"
+  "testing"
+
+  . "github.com/onsi/ginkgo"
+  . "github.com/onsi/gomega"
+)
+
+func TestProcessor(t *testing.T) {
+  log.SetOutput(ioutil.Discard)
+  RegisterFailHandler(Fail)
+  RunSpecs(t, "Processor Suite")
+}

data/ext/types/net.proto

@@ -0,0 +1,25 @@
+syntax = "proto3";
+package net;
+option go_package = "github.com/ukparliament/gromnative/ext/types/net";
+
+service Net {
+    rpc Get (GetInput) returns (GetOutput);
+}
+
+message GetInput {
+    string uri = 1;
+
+    message Header {
+        string key   = 1;
+        string value = 2;
+    }
+
+    repeated Header headers = 2;
+}
+
+message GetOutput {
+    string uri        = 1;
+    bytes  body       = 2;
+    int32  statusCode = 3;
+    string error      = 4;
+}